diff --git a/.fixtures.yml b/.fixtures.yml index 0f2bb67..9491f2f 100644 --- a/.fixtures.yml +++ b/.fixtures.yml @@ -1,5 +1,14 @@ fixtures: - repositories: - "stdlib": "git://github.com/puppetlabs/puppetlabs-stdlib.git" + forge_modules: + augeas_core: 'puppetlabs/augeas_core' + epel: + repo: 'puppet/epel' + ref: '3.0.1' + python: 'puppet/python' + stdlib: 'puppetlabs/stdlib' + vcsrepo: + repo: 'puppetlabs/vcsrepo' + ref: '3.2.1' + yumrepo_core: 'puppetlabs/yumrepo_core' symlinks: "yas3fs": "#{source_dir}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 32fe3c3..20c22d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,16 @@ All notable changes to this project will be documented in this file. -## Release 0.4.0 +## Release 0.5.0 **Features** +Add python3 and virtual environment support + **Bugfixes** **Known Issues** + +When not using virtual environment, please confirm install +location of yas3fs and make symlink from /usr/local/bin/yas3fs +if not created in /usr/local/bin diff --git a/README.md b/README.md index 4044437..5516325 100644 --- a/README.md +++ b/README.md @@ -32,71 +32,41 @@ invalidating caches on other nodes using SQS and SNS. * fuse package and configuration * init jobs (sysvinit, upstart or systemd) that are used to manage yas3fs mounts -* python-pip package (optional) +* python package ###Beginning with yas3fs -To install fuse, python-pip, and yas3fs -```puppet -class { 'yas3fs': } -``` -To install fuse, python-pip, and yas3fs from github source -```puppet -class { 'yas3fs': - provider: 'vcs', -} -``` - -If you'd rather provide pip though some other means, set -`install_pip_package` to `false`: +To install fuse and yas3fs from github source ```puppet class { 'yas3fs': - install_pip_package => false, } ``` - ##Usage ###Classes and Defined Types ####Class: `yas3fs` -The primary module. By default, fuse, python-pip, and yas3fs are installed and +The primary module. By default, fuse, and yas3fs are installed and configured. **Parameters within `yas3fs`** -#####`install_pip_package` - -When set to true, the python-pip package is installed. If the parameter is false -then the pip command should be provided by some other means or yas3fs will not -be installed. - #####`init_system` Defines the type of init script/configuration to install out of `upstart`, `systemd` or `sysvinit`. If the parameter is unset, autodiscovery takes place. +Default: $facts['service_provider'] -#####`provider` +#####`manage_python` -Sets if yas3fs should installed from default pip package provider -or if github source is pulled and pip package is built. -Set to 'vcs' to build from github source. -To switch to vcs from yas3fs previously installied pip package -manually intervention is required. Please manually -uninstall previosly installied pip package before switching to -vcs type install. -Default: 'pip' +Defines if this module should attempt to install and manage python +Default: false -#####`vcs_remote` +#####`manage_requirements`: -Optionally set yas3fs source code to be other than original -Default: 'https://github.com/danilop/yas3fs.git' - -#####`vcs_revision` - -Optionally set to use code from a specific commit or branch -Default: 'master' +Defines if this module should attempt to install and manage yas3fs dependencies. +Default: true #####`mounts` @@ -113,6 +83,30 @@ class { 'yas3fs': } } ``` +#####`python_version`: + +Which version /usr/bin/pythonX[.Y] should be used to run yas3fs? +Major version X require, minor version Y is optional. +i.g. 3,3.6,3.8,2,2.7 +Default: 3 + +#####`vcs_remote` + +Optionally set yas3fs source code to be other than original +Default: 'https://github.com/danilop/yas3fs.git' + +#####`vcs_revision` + +Optionally set to use code from a specific commit or branch +Default: '5bbf8296b5cb16c8afecad94ea55d03c4052a683' # v2.4.6 No tag available + +#####`venv_path` + +Path to python virtual environment. +If set to '' system python library path will be used. +Highly recommend installing yas3fs to a virtual environment +to avoid causing issues with your python based package +management systems i.g. apt, yum... ####Defined Type: `yas3fs::mount` @@ -132,6 +126,14 @@ yas3fs::mount { 'example-mount': **Parameters within `yas3fs::mount`** +#####`s3_url` + +The S3 URL that should be mounted (e.g. s3://my-bucket/my-path) + +#####`local_path` + +The location where the S3 bucket should be mounted. + #####`ensure` Control what to do with this mount. Valid values are `mounted` (default), `unmounted`, `absent`, @@ -140,24 +142,25 @@ and `present`. WARNING: setting ensure to `absent` removes the service configuration, but cannot verify that the service is stopped. -#####`s3_url` +#####`init_system` -The S3 URL that should be mounted (e.g. s3://my-bucket/my-path) +Your init system so that proper mount startup script can be created. -#####`local_path` +#####`options` -The location where the S3 bucket should be mounted. +An array of command line arguments that should be passed to yas3fs. The leading +dashes can be omitted. A full list of options is in the +[yas3fs documentation](https://github.com/danilop/yas3fs/blob/master/README.md). #####`aws_access_key_id` and `aws_secret_access_key` The credentials to use when connecting to AWS. Credentials can be omitted on EC2 instances with appropriate IAM roles assigned. -#####`options` +#####`venv_path` -An array of command line arguments that should be passed to yas3fs. The leading -dashes can be omitted. A full list of options is in the -[yas3fs documentation](https://github.com/danilop/yas3fs/blob/master/README.md). +The virtual environment where yas3fs is installed. +Help your init system start the mount properly. ##Reference @@ -170,7 +173,7 @@ dashes can be omitted. A full list of options is in the ####Private Classes * `yas3fs::config`: Manages the fuse configuration -* `yas3fs::package`: Installs pip, fuse, and yas3fs +* `yas3fs::install`: Installs fuse, and yas3fs * `yas3fs::params`: Manages base parameters ###Defined Types @@ -182,4 +185,4 @@ dashes can be omitted. A full list of options is in the ##Limitations yas3fs is written for python 2.6, though there has been a lot of success running -it on 2.7. +it on 2.7, and now 3.x. diff --git a/files/requirements.txt b/files/requirements.txt new file mode 100644 index 0000000..a785c7d --- /dev/null +++ b/files/requirements.txt @@ -0,0 +1,7 @@ +# This file is puppet managed +# any changes will be removed +# by puppet. +# Minimum requirements for yas3fs +setuptools>=2.2 +boto>=2.25.0 +boto3>=1.6.12 diff --git a/manifests/config.pp b/manifests/config.pp index 83000ae..bccba5c 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -2,11 +2,13 @@ class yas3fs::config { assert_private() - file { '/etc/fuse.conf': - ensure => present, - owner => 'root', - group => 'root', - mode => '0664', + if ! defined(File['/etc/fuse.conf']) { + file { '/etc/fuse.conf': + ensure => present, + owner => 'root', + group => 'root', + mode => '0664', + } } augeas { 'fuse.conf:user_allow_other': diff --git a/manifests/init.pp b/manifests/init.pp index 9081d39..baecbdb 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -1,15 +1,49 @@ # A module to manage S3 mounts using yas3fs +# * `init_system`: +# Specify the init system used. i.g. sysv, systemd +# +# * `manage_python`: +# Should this module attempt to install and manage +# python? +# +# * `manage_requirements`: +# Should this module attempt to install and manage +# yas3fs dependencies? +# +# * `mounts`: +# A hash of mounts and mount options passed to +# defined type yas3fs::mount +# +# * `python_version`: +# Which version /usr/bin/pythonX[.Y] should be +# used to run yas3fs? +# +# * `vcsrepo`: +# The VCS repository to fetch yas3fs code. +# +# * `vcs_revision`: +# Revision/Commit/Branch/Tag to check out +# when installing yas3fs +# +# * `venv_path`: +# If set to '' system python library path will be used. +# Highly recommend installing yas3fs to a virtual environment +# to avoid causing issues with your python based package +# management systems i.g. apt, yum... +# class yas3fs ( - $install_pip_package = $::yas3fs::params::install_pip_package, - $init_system = $::yas3fs::params::init_system, - $mounts = {}, - Enum['pip', 'vcs'] $provider = $::yas3fs::params::provider, - $vcs_remote = $::yas3fs::params::vcs_remote, - $vcs_revision = $::yas3fs::params::vcs_revision, + Enum['systemd','sysvinit','upstart'] $init_system = $::yas3fs::params::init_system, + Boolean $manage_python = false, + Boolean $manage_requirements = true, + Hash $mounts = {}, + String[1] $python_version = '3', # Versions 2,2.7,3,3.6 + String[9] $vcs_remote = 'https://github.com/danilop/yas3fs.git', + String[1] $vcs_revision = '5bbf8296b5cb16c8afecad94ea55d03c4052a683', # v2.4.6 No tag available + String $venv_path = '/opt/yas3fs/venv', #Path to install python virtual environment ) inherits yas3fs::params { anchor { 'yas3fs::begin': } - -> class { '::yas3fs::package': } + -> class { '::yas3fs::install': } -> class { '::yas3fs::config': } -> anchor { 'yas3fs::end':} diff --git a/manifests/install.pp b/manifests/install.pp new file mode 100644 index 0000000..a76bd6f --- /dev/null +++ b/manifests/install.pp @@ -0,0 +1,163 @@ +# yas3fs package install +# +# Class parameters are defined in init.pp +# Please check init.pp for usage. +# Please set parameters in base class (init.pp) +# +class yas3fs::install ( + $manage_python = $::yas3fs::manage_python, + $manage_requirements = $::yas3fs::manage_requirements, + $python_version = $::yas3fs::python_version, + $vcs_remote = $::yas3fs::vcs_remote, + $vcs_revision = $::yas3fs::vcs_revision, + $venv_path = $::yas3fs::venv_path, +){ + assert_private() + + + # python::pyvenv includes python class + # so if we do not want python to be managed/touched + # in any way manage_python should be set to false + if !defined(Class['python']) { + class { 'python': + version => $python_version, + manage_python_package => $manage_python, + manage_pip_package => $manage_python, + } + } + + # pyenv needs major and minor version number + # borrow code trick + # from https://github.com/voxpupuli/puppet-python/blob/v6.2.1/manifests/pyvenv.pp#L39-L42 + # python3_version and python2_version are facts created by python module + $_python_version = $python_version ? { + '3' => $facts['python3_version'], + '2' => $facts['python2_version'], + default => $python_version, + } + + if ($venv_path != '') { + if versioncmp($python_version, '3') <0 { + fail( "Virtual environment can only be used with python3+, please set venv_path to '' for use with python2") + } + + # Create all parent directories in provided $venv_path + # https://stackoverflow.com/a/56909439 + $venv_path_dirs = $venv_path[1,-1].dirname.split('/').reduce([]) |$memo, $subdir| { + $_dir = $memo.empty ? { + true => "/${subdir}", + default => "${$memo[-1]}/${subdir}", + } + concat($memo, $_dir) + } + file {$venv_path_dirs: + ensure => directory, + } + + python::pyvenv { 'yas3fs virtual environment' : + ensure => present, + version => $_python_version, + systempkgs => false, + venv_dir => $venv_path, + before => Exec['install yas3fs'], + } + } + + if ! defined(Package['fuse']) { + package { 'fuse': + ensure => present, + allow_virtual => true + } + } + + if ($facts['os']['family'] == 'RedHat') { + if ! defined(Package['fuse-libs']) { + package { 'fuse-libs': + ensure => present, + allow_virtual => true + } + } + } + + # Unfortunately puppet package resource does not allow us to + # Uninstall a specific version of a pip package. + # Attempting to set ensure absent for pip package yas3fs + # Will also uninstall the version installed from source. + # Attempting to set ensure latest also does not work well. + # Please manually uninstall yas3fs pip package if going + # from pip package to install from source. + + # On Redhat 7 + # yum installed python-setuptools is 0.9.8 + # can not uninstall python-setuptools due to dependencies, + # but can let pip try to install a newer version overtop + # howver not recommended, use virtual environment instead. + # yas3fs setup.py barfs on setuptools and boto3 installs. + # we will use python::requirements to ensure + # yas3fs requirements are installed. + + $virtualenv = $venv_path ? { + '' => undef, + default => $venv_path, + } + + if ($manage_requirements == true) { + file { '/root/yas3fs_requirements.txt' : + ensure => present, + content => file('yas3fs/requirements.txt'), + owner => 'root', + group => 'root', + mode => '0664', + notify => Python::Requirements['/root/yas3fs_requirements.txt'] + } + python::requirements { '/root/yas3fs_requirements.txt' : + virtualenv => $virtualenv, + require => File['/root/yas3fs_requirements.txt'], + before => Exec['install yas3fs'], + } + } + + vcsrepo { '/var/tmp/yas3fs': + # Just 'present' so we do not beatup our git repository + # provider every 30mins + ensure => present, + provider => git, + source => $vcs_remote, + revision => $vcs_revision, + } + + + # TODO offer better cleanup, and remove yas3fs via pip + # before install yas3fs overtop of previous version + + #If Virtual Environment created pythin should be symlinked to $python_version + if $venv_path != '' { + $_exec_command = "source ${venv_path}/bin/activate && python${python_version} /var/tmp/yas3fs/setup.py install --prefix=${venv_path}" + $_exec_creates = "${venv_path}/bin/yas3fs" + }else{ + $_exec_command = "/usr/bin/env python${python_version} /var/tmp/yas3fs/setup.py install" + # Newer python setup tools installs to /usr/local/bin/ ??? + # Users which install does not create /usr/local/bin/yasfs consider making a symlink to + # Wherever it got installed so puppet will not attempt to reinstall every run + # I can really only test on a Redhat machine and relay on rspec to test everything else + # - Ron (mojibake-umd) - + $_exec_creates = '/usr/local/bin/yas3fs' + } + + # Trigger install of yas3fs on vcsrepo refresh + exec { 'remove install yas3fs creates file': + refreshonly => true, + command => "/usr/bin/rm ${_exec_creates}", + subscribe => Vcsrepo['/var/tmp/yas3fs'], + onlyif => "test -f ${_exec_creates}", + notify => Exec['install yas3fs'], + } + + exec { 'install yas3fs': + command => $_exec_command, + creates => $_exec_creates, + cwd => '/var/tmp/yas3fs', + provider => 'shell', + require => Vcsrepo['/var/tmp/yas3fs'], + } +} diff --git a/manifests/mount.pp b/manifests/mount.pp index 7ced8bf..3b87fba 100644 --- a/manifests/mount.pp +++ b/manifests/mount.pp @@ -1,11 +1,49 @@ # A yas3fs mount +# Defined type to create one or more mount points +# to S3 bucket endpoints +# +# * `s3_url`: +# Full S3 url path +# i.g. s3://my-bucket/my-path/ +# +# * `local_path` +# The path on the system that S3 bucket +# contents should be mount +# i.g. /mnt/bucket_here +# +# * `ensure` +# Define the state of the S3 mount +# Validate options: +# mounted,unmounted,present,absent +# +# * `options` +# Array of command line options to passed +# to yas3fs at mount time. +# Full list of options available at +# https://github.com/danilop/yas3fs/blob/master/README.md#full-usage +# +# * `aws_access_key_id` +# Access key id of IAM user with IAM policies +# granting access to the S3 bucket defined in s3_url +# +# * `aws_access_access_key` +# Secret access key of Access key id defined above +# Omitting Access key id and Secret access key will +# let yas3fs rely on IAM Profile of EC2 instance. +# +# * `venv_path` +# The virtual environment where yas3fs is installed. +# Help your init system start the mount properly. +# define yas3fs::mount ( - $s3_url, - $local_path, - $ensure = 'mounted', - $options = [], - $aws_access_key_id = undef, - $aws_secret_access_key = undef, + String[6] $s3_url, + String[1] $local_path, + Enum['mounted','unmounted','present','absent'] $ensure = 'mounted', + Enum['systemd','sysvinit','upstart'] $init_system = $::yas3fs::init_system, + Array $options = [], + Optional[String] $aws_access_key_id = undef, + Optional[String] $aws_secret_access_key = undef, + String $venv_path = $::yas3fs::venv_path, ) { validate_array($options) validate_string($s3_url, $local_path, $aws_access_key_id, $aws_secret_access_key) @@ -40,12 +78,14 @@ } } - case $yas3fs::init_system { + case $init_system { 'systemd': { exec { "yas3fs_reload_systemd-${name}": # SystemD needs a reload after any unit file change command => 'systemctl daemon-reload', - path => ['/bin', '/sbin', '/usr/bin', '/usr/sbin'], + # Thank @djtaylor https://github.com/pcfens/puppet-yas3fs/pull/5 + # /usr/bin/env yas3fs + path => ['/bin', '/sbin', '/usr/bin', '/usr/sbin', '/usr/local/bin'], refreshonly => true, subscribe => File["yas3fs-${name}"], before => Service["s3fs-${name}"], @@ -83,7 +123,7 @@ } } default : { - fail("Unknown init system ${yas3fs::init_system}, unable to install startup script for Yas3fs") + fail("Unknown init system ${init_system}, unable to install startup script for Yas3fs") } } diff --git a/manifests/package.pp b/manifests/package.pp deleted file mode 100644 index 8d3d299..0000000 --- a/manifests/package.pp +++ /dev/null @@ -1,81 +0,0 @@ -# yas3fs package install -class yas3fs::package ( - $provider = $::yas3fs::provider, - $vcs_remote = $::yas3fs::vcs_remote, - $vcs_revision = $::yas3fs::vcs_revision, -){ - assert_private() - - if $::yas3fs::install_pip_package { - package { 'python-pip': - ensure => present, - allow_virtual => true - } - } - - package { 'fuse': - ensure => present, - allow_virtual => true - } - - if ($::osfamily == 'RedHat') { - package { 'fuse-libs': - ensure => present, - allow_virtual => true - } - } - - case $provider { - 'pip': { - package { 'yas3fs': - ensure => present, - provider => 'pip', - allow_virtual => true - } - } - 'vcs': { - # Unfortunately puppet package resource does not allow us to - # Uninstall a specific version of a pip package. - # Attempting to set ensure absent for pip package yas3fs - # Will also uninstall the version installed from source. - # Attempting to set ensure latest also does not work well. - # Please manually uninstall yas3fs pip package if going - # pip package to install from source. - - # yas3fs setup.py barfs on setuptools and boto3 installs. - # I think it is RHEL's fault - package { 'setuptools': - ensure => '2.2', - provider => 'pip', - allow_virtual => true - } - package { 'boto3': - ensure => 'present', - provider => 'pip', - allow_virtual => true - } - vcsrepo { '/var/tmp/yas3fs': - # Just 'present' so we do not beatup our git repository - # provider every 30mins - ensure => present, - provider => git, - source => $vcs_remote, - revision => $vcs_revision, - } - - exec { 'install yas3fs': - command => 'python /var/tmp/yas3fs/setup.py install', - creates => '/usr/bin/yas3fs', - cwd => '/var/tmp/yas3fs', - require => [Package['setuptools', 'boto3'],Vcsrepo['/var/tmp/yas3fs']], - } - } - default: { - package { 'yas3fs': - ensure => present, - provider => 'pip', - allow_virtual => true - } - } - } -} diff --git a/manifests/params.pp b/manifests/params.pp index 8ac5e15..a56c976 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -1,16 +1,8 @@ # yas3fs defaults class yas3fs::params { - $install_pip_package = true - $provider = 'pip' - $vcs_remote = 'https://github.com/danilop/yas3fs.git' - $vcs_revision = 'master' - # Use the jethrocarr-initfact module to determine which init system is in - # use, fall back to using upstart if missing to ensure compatibility with - # exiting puppet-yas3fs users. - - if ($::initsystem) { - $init_system = $::initsystem + if ($facts['service_provider']) { + $init_system = $facts['service_provider'] } else { $init_system = 'upstart' } diff --git a/metadata.json b/metadata.json index 0a92ff1..f8327f5 100644 --- a/metadata.json +++ b/metadata.json @@ -1,6 +1,6 @@ { "name": "pcfens-yas3fs", - "version": "0.4.0", + "version": "1.0.0", "author": "pcfens", "summary": "A module to install and manage yas3fs mounts", "license": "Apache-2.0", @@ -10,27 +10,25 @@ "dependencies": [ { "name": "puppetlabs/stdlib", - "version_requirement": ">=4.6.0 <5.0.0" - }, - { - "name": "jethrocarr/initfact", - "version_requirement": ">=1.0.0 <2.0.0" + "version_requirement": ">=4.6.0 <7.0.0" } ], "operatingsystem_support": [ { "operatingsystem": "RedHat", "operatingsystemrelease": [ - "5.0", - "6.0", - "7.0" + "6", + "7", + "8" ] }, { "operatingsystem": "Ubuntu", "operatingsystemrelease": [ - "12.04", - "14.04" + "16.04", + "18.04", + "20.04", + "22.04" ] } ], @@ -38,6 +36,10 @@ { "name": "puppet", "version_requirement": ">= 6.21.0 < 8.0.0" + }, + { + "name": "vcsrepo", + "version_requirement": ">=1.0.0 < 4.0.0" } ], "tags": [ diff --git a/pdk.yaml b/pdk.yaml new file mode 100644 index 0000000..4bef4bd --- /dev/null +++ b/pdk.yaml @@ -0,0 +1,2 @@ +--- +ignore: [] diff --git a/spec/classes/yas3fs_spec.rb b/spec/classes/yas3fs_spec.rb index 4986487..5d40a27 100644 --- a/spec/classes/yas3fs_spec.rb +++ b/spec/classes/yas3fs_spec.rb @@ -1,58 +1,334 @@ require 'spec_helper' -describe 'yas3fs', :type => :class do - - let :facts do { - :osfamily => 'Debian', - :initsystem => 'upstart' - } end - - - context 'defaults' do - it { is_expected.to contain_package('fuse') } - it { is_expected.to contain_package('python-pip') } - - it { is_expected.to contain_augeas('fuse.conf:user_allow_other').with( - 'context' => '/files/etc/fuse.conf', - 'changes' => [ - 'set user_allow_other ""', - ] - ) } - - context 'with install_pip_package set to false' do - let :params do - { - :install_pip_package => false, - } - end +describe 'yas3fs', type: :class do + context 'on a supported operating system' do + on_supported_os.each do |os, facts| + context os do + let(:facts) do + facts.merge({ + python3_version: '3.6', + python2_version: '2.7', - it { is_expected.not_to contain_package('python-pip')} + }) + end - end + context 'with defaults' do + describe 'Is expected to install/configure package dependencies and yas3fs' do + it { + is_expected.to contain_class('python').with( + 'version' => '3', + 'manage_python_package' => false, + 'manage_pip_package' => false, + ) + } + + it { is_expected.to contain_file('/opt/yas3fs') } + it { is_expected.to contain_file('/opt/yas3fs/venv') } + it { + is_expected.to contain_python__pyvenv('yas3fs virtual environment').with( + 'ensure' => 'present', + 'version' => '3.6', + 'systempkgs' => false, + 'venv_dir' => '/opt/yas3fs/venv', + 'before' => 'Exec[install yas3fs]', + ) + } + + it { is_expected.to contain_package('fuse') } + # This case statement currently does not work running pdk test unit + # I think it has something to newer facter/facterdb gem + # attempting to run facterdb locally complains of switching + # from get_os_facts to get_facts. + case facts[:os][:family] + when 'Redhat' + it do + is_expected.to contain_package('fuse-libs') + end + end + + it { + is_expected.to contain_augeas('fuse.conf:user_allow_other').with( + 'context' => '/files/etc/fuse.conf', + 'changes' => [ + 'set user_allow_other ""', + ], + ) + } + it { + is_expected.to contain_file('/root/yas3fs_requirements.txt').with( + 'ensure' => 'present', + 'owner' => 'root', + 'group' => 'root', + 'mode' => '0664', + ) + } + + it { + is_expected.to contain_python__requirements('/root/yas3fs_requirements.txt').with( + 'virtualenv' => '/opt/yas3fs/venv', + 'require' => 'File[/root/yas3fs_requirements.txt]', + 'before' => 'Exec[install yas3fs]', + ) + } + + it { + is_expected.to contain_vcsrepo('/var/tmp/yas3fs').with( + 'ensure' => 'present', + 'provider' => 'git', + 'source' => 'https://github.com/danilop/yas3fs.git', + 'revision' => '5bbf8296b5cb16c8afecad94ea55d03c4052a683', + ) + } + + it { + is_expected.to contain_exec('remove install yas3fs creates file').with( + 'refreshonly' => true, + 'command' => '/usr/bin/rm /opt/yas3fs/venv/bin/yas3fs', + 'subscribe' => 'Vcsrepo[/var/tmp/yas3fs]', + 'notify' => 'Exec[install yas3fs]', + ) + } + it { + is_expected.to contain_exec('install yas3fs').with( + 'command' => 'source /opt/yas3fs/venv/bin/activate && python3 /var/tmp/yas3fs/setup.py install --prefix=/opt/yas3fs/venv', + 'creates' => '/opt/yas3fs/venv/bin/yas3fs', + 'cwd' => '/var/tmp/yas3fs', + 'require' => 'Vcsrepo[/var/tmp/yas3fs]', + ) + } + end + + context 'with mounts in the initial resource creation' do + let :params do + { + mounts: { + 'test-mount' => { + 's3_url' => 's3://example-bucket/', + 'local_path' => '/media/s3', + } + } + } + end + + it { + is_expected.to contain_file('yas3fs-test-mount').with( + 'ensure' => 'present', + 'path' => '/etc/init/s3fs-test-mount.conf', + ) + } - context 'with mounts in the initial resource creation' do - let :params do - { - :mounts => { - 'test-mount' => { - 's3_url' => 's3://example-bucket/', - 'local_path' => '/media/s3', + it { + is_expected.to contain_service('s3fs-test-mount').with( + 'ensure' => 'running', + 'enable' => true, + ) } + end + end + + context 'with venv_path set to \'\'' do + let(:params) do + { + 'venv_path' => '', + } + end + + case facts[:os][:family] + when 'Redhat', 'Amazon' + case facts[:os][:release][:major] + when '6', '7' + it { + is_expected.to contain_python__requirements('/root/yas3fs_requirements.txt').with( + 'virtualenv' => 'system', + 'require' => 'File[/root/yas3fs_requirements.txt]', + 'before' => 'Exec[install yas3fs]', + ) + } + end + end + it { + is_expected.to contain_exec('remove install yas3fs creates file').with( + 'refreshonly' => true, + 'command' => '/usr/bin/rm /usr/local/bin/yas3fs', + 'subscribe' => 'Vcsrepo[/var/tmp/yas3fs]', + 'notify' => 'Exec[install yas3fs]', + ) } - } - end + it { + is_expected.to contain_exec('install yas3fs').with( + 'command' => '/usr/bin/env python3 /var/tmp/yas3fs/setup.py install', + 'creates' => '/usr/local/bin/yas3fs', + 'cwd' => '/var/tmp/yas3fs', + 'require' => 'Vcsrepo[/var/tmp/yas3fs]', + ) + } + end - it { is_expected.to contain_file('yas3fs-test-mount').with( - 'ensure' => 'present', - 'path' => '/etc/init/s3fs-test-mount.conf', - ) } + context 'with venv_path set to /some/deep/dark/path/venv' do + let(:params) do + { + 'venv_path' => '/some/deep/dark/path/venv', + } + end - it { is_expected.to contain_service('s3fs-test-mount').with( - 'ensure' => 'running', - 'enable' => true, - ) } + it { is_expected.to contain_file('/some') } + it { is_expected.to contain_file('/some/deep') } + it { is_expected.to contain_file('/some/deep/dark') } + it { is_expected.to contain_file('/some/deep/dark/path') } + it { is_expected.to contain_file('/some/deep/dark/path/venv') } + end + context 'with manage_python set to true' do + let(:params) do + { + 'manage_python' => true, + } + end - end + it { + is_expected.to contain_class('python').with( + 'version' => '3', + 'manage_python_package' => true, + 'manage_pip_package' => true, + ) + } + end + + context 'with manage_requirements set to false' do + let(:params) do + { + 'manage_requirements' => false, + } + end + it { + is_expected.not_to contain_python__requirements('/root/yas3fs_requirements.txt') + } + end + + context 'with python_version = 2' do + let(:params) do + { + 'python_version' => '2', + 'venv_path' => '', + } + end + + it { + is_expected.to contain_class('python').with( + 'version' => '2', + 'manage_python_package' => false, + 'manage_pip_package' => false, + ) + } + it { + is_expected.to contain_exec('install yas3fs').with( + 'command' => '/usr/bin/env python2 /var/tmp/yas3fs/setup.py install', + 'creates' => '/usr/local/bin/yas3fs', + 'cwd' => '/var/tmp/yas3fs', + 'require' => 'Vcsrepo[/var/tmp/yas3fs]', + ) + } + end + + context 'with python_version = 2.7' do + let(:params) do + { + 'python_version' => '2.7', + 'venv_path' => '', + } + end + + it { + is_expected.to contain_class('python').with( + 'version' => '2.7', + 'manage_python_package' => false, + 'manage_pip_package' => false, + ) + } + it { + is_expected.to contain_exec('install yas3fs').with( + 'command' => '/usr/bin/env python2.7 /var/tmp/yas3fs/setup.py install', + 'creates' => '/usr/local/bin/yas3fs', + 'cwd' => '/var/tmp/yas3fs', + 'require' => 'Vcsrepo[/var/tmp/yas3fs]', + ) + } + end + + context 'with python_version = 2.7 and venv_path = /opt/yas3fs/venv (default)' do + let(:params) do + { + 'python_version' => '2.7', + } + end + + it { + is_expected.to compile.and_raise_error(%r{Virtual environment can only be used with python3}) + } + end + + context 'with python_version = python3' do + let(:params) do + { + 'python_version' => '3', + } + end + + it { + is_expected.to contain_class('python').with( + 'version' => '3', + 'manage_python_package' => false, + 'manage_pip_package' => false, + ) + } + it { + is_expected.to contain_python__requirements('/root/yas3fs_requirements.txt').with( + 'virtualenv' => '/opt/yas3fs/venv', + 'require' => 'File[/root/yas3fs_requirements.txt]', + 'before' => 'Exec[install yas3fs]', + ) + } + it { + is_expected.to contain_exec('install yas3fs').with( + 'command' => 'source /opt/yas3fs/venv/bin/activate && python3 /var/tmp/yas3fs/setup.py install --prefix=/opt/yas3fs/venv', + 'creates' => '/opt/yas3fs/venv/bin/yas3fs', + 'cwd' => '/var/tmp/yas3fs', + 'require' => 'Vcsrepo[/var/tmp/yas3fs]', + ) + } + end + + context 'with python_version = 3.6' do + let(:params) do + { + 'python_version' => '3.6', + } + end + + it { + is_expected.to contain_class('python').with( + 'version' => '3.6', + 'manage_python_package' => false, + 'manage_pip_package' => false, + ) + } + it { + is_expected.to contain_python__requirements('/root/yas3fs_requirements.txt').with( + 'virtualenv' => '/opt/yas3fs/venv', + 'require' => 'File[/root/yas3fs_requirements.txt]', + 'before' => 'Exec[install yas3fs]', + ) + } + it { + is_expected.to contain_exec('install yas3fs').with( + 'command' => 'source /opt/yas3fs/venv/bin/activate && python3.6 /var/tmp/yas3fs/setup.py install --prefix=/opt/yas3fs/venv', + 'creates' => '/opt/yas3fs/venv/bin/yas3fs', + 'cwd' => '/var/tmp/yas3fs', + 'require' => 'Vcsrepo[/var/tmp/yas3fs]', + ) + } + end + end + end end end diff --git a/spec/defines/mount_spec.rb b/spec/defines/mount_spec.rb index e008865..02c6d57 100644 --- a/spec/defines/mount_spec.rb +++ b/spec/defines/mount_spec.rb @@ -1,7 +1,6 @@ require 'spec_helper' -describe 'yas3fs::mount', :type => :define do - +describe 'yas3fs::mount', type: :define do let :pre_condition do 'include ::yas3fs' end @@ -9,81 +8,331 @@ let :title do 'test-mount' end + let(:facts) do + facts + end + + describe 'on Redhat systems' do + context 'with upstart <= Rhel6' do + let :params do + { + s3_url: 's3://test-bucket', + local_path: '/media/test-mount', + } + end + + let :facts do + { + service_provider: 'upstart', + osfamily: 'Redhat', + os: { + family: 'RedHat', + hardware: 'x86_64', + name: 'RedHat', + release: { + full: '6.10', + major: '6', + minor: '10' + }, + }, + python3_version: '3.6', + python2_version: '2.7', + } + end + + it { + is_expected.to contain_file('yas3fs-test-mount').with( + 'ensure' => 'present', + 'path' => '/etc/init/s3fs-test-mount.conf', + 'notify' => 'Service[s3fs-test-mount]', + 'content' => %r{exec /opt/yas3fs/venv/bin/yas3fs -f \"\$S3_URL\" \"\$LOCAL_PATH\"}, + ) + } + + it { + is_expected.to contain_service('s3fs-test-mount').with( + 'ensure' => 'running', + 'enable' => true, + ) + } + end - describe 'on all systems' do - context 'with upstart' do - let :params do { - :s3_url => 's3://test-bucket', - :local_path => '/media/test-mount', - } end + context 'with upstart and venv_path set to \'\'' do + let :params do + { + s3_url: 's3://test-bucket', + local_path: '/media/test-mount', + venv_path: '', + } + end - let :facts do { - :initsystem => 'upstart', - :osfamily => 'Debian' - } end + let :facts do + { + service_provider: 'upstart', + osfamily: 'Redhat', + os: { + family: 'RedHat', + hardware: 'x86_64', + name: 'RedHat', + release: { + full: '6.10', + major: '6', + minor: '10' + }, + }, + python3_version: '3.6', + python2_version: '2.7', + } + end - it { is_expected.to contain_file('yas3fs-test-mount').with( + it { + is_expected.to contain_file('yas3fs-test-mount').with( 'ensure' => 'present', 'path' => '/etc/init/s3fs-test-mount.conf', 'notify' => 'Service[s3fs-test-mount]', - ) } + 'content' => %r{exec /usr/bin/env yas3fs -f \"\$S3_URL\" \"\$LOCAL_PATH\"}, + ) + } - it { is_expected.to contain_service('s3fs-test-mount').with( + it { + is_expected.to contain_service('s3fs-test-mount').with( 'ensure' => 'running', 'enable' => true, - ) } + ) + } end - context 'with systemd' do - let :facts do { - :initsystem => 'systemd', - :osfamily => 'Debian' - } end + context 'with systemd >= Rhel7' do + let :facts do + { + service_provider: 'systemd', + osfamily: 'Redhat', + os: { + family: 'RedHat', + hardware: 'x86_64', + name: 'RedHat', + release: { + full: '7.9', + major: '7', + minor: '9' + }, + }, + python3_version: '3.6', + python2_version: '2.7', + } + end - let :params do { - :s3_url => 's3://test-bucket', - :local_path => '/media/test-mount', - } end + let :params do + { + s3_url: 's3://test-bucket', + local_path: '/media/test-mount', + } + end - it { is_expected.to contain_exec('yas3fs_reload_systemd-test-mount').with( + it { + is_expected.to contain_exec('yas3fs_reload_systemd-test-mount').with( 'command' => 'systemctl daemon-reload', 'subscribe' => 'File[yas3fs-test-mount]', 'before' => 'Service[s3fs-test-mount]', - ) } + ) + } + + it { + is_expected.to contain_file('yas3fs-test-mount').with( + 'ensure' => 'present', + 'path' => '/etc/systemd/system/s3fs-test-mount.service', + 'notify' => 'Service[s3fs-test-mount]', + 'content' => %r{ExecStart=/opt/yas3fs/venv/bin/yas3fs -f s3://test-bucket /media/test-mount}, + ) + } + it { + is_expected.to contain_service('s3fs-test-mount').with( + 'ensure' => 'running', + 'enable' => true, + ) + } + end - it { is_expected.to contain_file('yas3fs-test-mount').with( + context 'with systemd and venv_path set to \'\'' do + let :facts do + { + service_provider: 'systemd', + osfamily: 'Redhat', + os: { + family: 'RedHat', + hardware: 'x86_64', + name: 'RedHat', + release: { + full: '7.9', + major: '7', + minor: '9' + }, + }, + python3_version: '3.6', + python2_version: '2.7', + } + end + + let :params do + { + s3_url: 's3://test-bucket', + local_path: '/media/test-mount', + venv_path: '', + } + end + + it { + is_expected.to contain_file('yas3fs-test-mount').with( + 'content' => %r{ExecStart=/usr/bin/env yas3fs -f s3://test-bucket /media/test-mount}, + ) + } + end + end + + describe 'on Ubuntu systems' do + context 'with systemd >= 16.04' do + let :facts do + { + service_provider: 'systemd', + osfamily: 'Debian', + os: { + family: 'Debian', + hardware: 'x86_64', + name: 'Ubuntu', + release: { + major: '20', + }, + }, + python3_version: '3.6', + python2_version: '2.7', + } + end + + let :params do + { + s3_url: 's3://test-bucket', + local_path: '/media/test-mount', + } + end + + it { + is_expected.to contain_exec('yas3fs_reload_systemd-test-mount').with( + 'command' => 'systemctl daemon-reload', + 'subscribe' => 'File[yas3fs-test-mount]', + 'before' => 'Service[s3fs-test-mount]', + ) + } + + it { + is_expected.to contain_file('yas3fs-test-mount').with( 'ensure' => 'present', 'path' => '/etc/systemd/system/s3fs-test-mount.service', 'notify' => 'Service[s3fs-test-mount]', - ) } + 'content' => %r{ExecStart=/opt/yas3fs/venv/bin/yas3fs -f s3://test-bucket /media/test-mount}, + ) + } - it { is_expected.to contain_service('s3fs-test-mount').with( + it { + is_expected.to contain_service('s3fs-test-mount').with( 'ensure' => 'running', 'enable' => true, - ) } + ) + } end + context 'with systemd and venv_path set to /opt/yas3fs/venv' do + let :facts do + { + service_provider: 'systemd', + osfamily: 'Debian', + os: { + family: 'Debian', + hardware: 'x86_64', + name: 'Ubuntu', + release: { + major: '20', + }, + }, + python3_version: '3.6', + python2_version: '2.7', + } + end + + let :params do + { + s3_url: 's3://test-bucket', + local_path: '/media/test-mount', + venv_path: '/opt/yas3fs/venv', + } + end + + it { + is_expected.to contain_file('yas3fs-test-mount').with( + 'content' => %r{/opt/yas3fs/venv/bin/yas3fs -f s3://test-bucket /media/test-mount}, + ) + } + end + end + + describe 'on Esoteric Debian sysvinit system' do context 'with sysvinit' do - let :facts do { - :initsystem => 'sysvinit', - :osfamily => 'Debian' - } end + let :facts do + { + service_provider: 'sysvinit', + osfamily: 'Debian', + os: { family: 'Debian' }, + python3_version: '3.6', + python2_version: '2.7', + } + end - let :params do { - :s3_url => 's3://test-bucket', - :local_path => '/media/test-mount', - } end + let :params do + { + s3_url: 's3://test-bucket', + local_path: '/media/test-mount', + } + end - it { is_expected.to contain_file('yas3fs-test-mount').with( - 'ensure' => 'present', - 'path' => '/etc/init.d/s3fs-test-mount', - 'notify' => 'Service[s3fs-test-mount]', - ) } + it { + is_expected.to contain_file('yas3fs-test-mount').with( + 'ensure' => 'present', + 'path' => '/etc/init.d/s3fs-test-mount', + 'notify' => 'Service[s3fs-test-mount]', + 'content' => %r{PATH=/opt/yas3fs/venv/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:\$PATH}, + ) + } - it { is_expected.to contain_service('s3fs-test-mount').with( + it { + is_expected.to contain_service('s3fs-test-mount').with( 'ensure' => 'running', 'enable' => true, - ) } + ) + } + end + context 'with sysvinit and venv_path set to \'\'' do + let :facts do + { + service_provider: 'sysvinit', + osfamily: 'Debian', + os: { family: 'Debian' }, + python3_version: '3.6', + python2_version: '2.7', + } + end + + let :params do + { + s3_url: 's3://test-bucket', + local_path: '/media/test-mount', + venv_path: '', + } + end + + it { + is_expected.to contain_file('yas3fs-test-mount').with( + 'content' => %r{PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:\$PATH}, + ) + } end end end diff --git a/templates/systemd.erb b/templates/systemd.erb index 45fae0d..e35a60b 100644 --- a/templates/systemd.erb +++ b/templates/systemd.erb @@ -13,11 +13,17 @@ Environment=AWS_ACCESS_KEY_ID=<%= @aws_access_key_id %> AWS_SECRET_ACCESS_KEY=<% <% @options.each do |v| -%> <% cmd_args = "#{cmd_args} --#{v}" -%> <% end -%> -# TODO: systemd requires a fully qualified path... but Puppet doesn't give us a -# good way to query what the path to a particular binary is :-/ This may need -# some further work, PRs welcome -ExecStart=/usr/bin/yas3fs -f <%= cmd_args %> <%= @s3_url %> <%= @local_path %> -ExecStop=/bin/umount <%= @local_path %> +# Thank @djtaylor https://github.com/pcfens/puppet-yas3fs/pull/5 +# /usr/bin/env yas3fs +<% if @venv_path != '' -%> + WorkingDirectory=<%= @venv_path %> + Environment=PATH=<%= @venv_path %>/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin + ExecStart=<%= @venv_path %>/bin/yas3fs -f <%= cmd_args %> <%= @s3_url %> <%= @local_path %> +<% else %> + Environment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin + ExecStart=/usr/bin/env yas3fs -f <%= cmd_args %> <%= @s3_url %> <%= @local_path %> +<% end %> +ExecStop=/usr/bin/env umount <%= @local_path %> RestartSec=5 Restart=on-failure OOMScoreAdjust=-1000 diff --git a/templates/sysvinit.erb b/templates/sysvinit.erb index 72437c0..4f748bc 100644 --- a/templates/sysvinit.erb +++ b/templates/sysvinit.erb @@ -13,7 +13,11 @@ # Description: Sets up a FUSE mount for an S3-backed filesystem using Yas3fs ### END INIT INFO -PATH=$PATH:/usr/local/bin +<% if @venv_path != '' -%> + PATH=<%= @venv_path %>/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH +<% else %> +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH +<% end %> export S3_URL="<%= @s3_url %>" export LOCAL_PATH="<%= @local_path %>" diff --git a/templates/upstart.erb b/templates/upstart.erb index 39e20ee..3cfa6f7 100644 --- a/templates/upstart.erb +++ b/templates/upstart.erb @@ -22,7 +22,11 @@ script if [ $MOUNTED = "1" ]; then umount "$LOCAL_PATH" fi - <%= "exec /usr/local/bin/yas3fs -f #{cmd_args} \"$S3_URL\" \"$LOCAL_PATH\"" %> +<% if @venv_path != '' -%> + <%= "exec #{@venv_path}/bin/yas3fs -f #{cmd_args} \"$S3_URL\" \"$LOCAL_PATH\"" %> +<% else %> + <%= "exec /usr/bin/env yas3fs -f #{cmd_args} \"$S3_URL\" \"$LOCAL_PATH\"" %> +<% end %> end script pre-stop script