diff --git a/.travis.yml b/.travis.yml index 812bf1644..77de91599 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,25 @@ -sudo: required - language: python - -## allows 'docker' command -services: - - docker +dist: trusty +sudo: required ## environment variables env: global: - - DOCKER_VERSION=1.12.3-0~trusty + - DOCKER_VERSION=17.03.0~ce-0~ubuntu-trusty ## build docker, install puppet (i.e. puppet parser validate) +## +## Note: we install docker per dockers official documentation: +## +## https://docs.docker.com/engine/installation/linux/ubuntu/ +## before_install: - # docker version before update - - docker version - - # list docker-engine versions - #- apt-cache madison docker-engine - - # upgrade docker-engine to specific version - - sudo apt-get -o Dpkg::Options::="--force-confnew" install -y docker-engine=${DOCKER_VERSION} - - # docker version after update - - docker version + # install docker to specific version + - sudo apt-get install apt-transport-https ca-certificates curl software-properties-common + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get install docker-ce=${DOCKER_VERSION} # install puppet - wget https://apt.puppetlabs.com/puppetlabs-release-pc1-trusty.deb diff --git a/Vagrantfile b/Vagrantfile index 8f6d3e339..c4e29c239 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,14 +1,14 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : +## -*- mode: ruby -*- +## vi: set ft=ruby : -# All Vagrant configuration is done below. The "2" in Vagrant.configure -# configures the configuration version (we support older styles for +## All Vagrant configuration is done below. The "2" in Vagrant.configure +## configures the configuration version (we support older styles for # backwards compatibility). Please don't change it unless you know what -# you're doing. +## you're doing. Vagrant.configure(2) do |config| - # The most common configuration options are documented and commented below. - # For a complete reference, please see the online documentation at - # https://docs.vagrantup.com. + ## The most common configuration options are documented and commented below. + ## For a complete reference, please see the online documentation at + ## https://docs.vagrantup.com. ## Variables (ruby syntax) atlas_repo = 'jeff1evesque' @@ -39,7 +39,7 @@ Vagrant.configure(2) do |config| end ## Every Vagrant development environment requires a box. You can search for - # boxes at https://atlas.hashicorp.com/search. + ## boxes at https://atlas.hashicorp.com/search. config.vm.box = "#{atlas_repo}/#{atlas_box}" config.vm.box_download_checksum = 'c26da6ba1c169bdc6e9168125ddb0525' config.vm.box_url = "https://atlas.hashicorp.com/#{atlas_repo}/boxes/#{atlas_box}/versions/#{box_version}/providers/virtualbox.box" @@ -49,8 +49,8 @@ Vagrant.configure(2) do |config| config.puppet_install.puppet_version = '4.9.3' ## Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. + ## within the machine from a port on the host machine. In the example below, + ## accessing "localhost:8080" will access port 80 on the guest machine. config.vm.network 'forwarded_port', guest: 5000, host: 8080 config.vm.network 'forwarded_port', guest: 443, host: 8585 @@ -64,8 +64,9 @@ Vagrant.configure(2) do |config| config.r10k.puppetfile_path = "puppet/environment/#{environment}/Puppetfile" ## Custom Manifest: install needed packages - # - # Note: future parser allow array iteration in the puppet manifest + ## + ## Note: future parser allow array iteration in the puppet manifest + ## config.vm.provision 'puppet' do |puppet| puppet.environment_path = 'puppet/environment' puppet.environment = environment @@ -74,110 +75,11 @@ Vagrant.configure(2) do |config| "puppet/environment/#{environment}/modules_contrib", "puppet/environment/#{environment}/modules", ] - puppet.manifest_file = 'install_packages.pp' + puppet.manifest_file = 'site.pp' puppet.hiera_config_path = 'hiera.yaml' end - ## Custom Manifest: build scikit-learn - config.vm.provision 'puppet' do |puppet| - puppet.environment_path = 'puppet/environment' - puppet.environment = environment - puppet.manifests_path = "puppet/environment/#{environment}/manifests" - puppet.module_path = [ - "puppet/environment/#{environment}/modules_contrib", - "puppet/environment/#{environment}/modules", - ] - puppet.manifest_file = 'install_sklearn.pp' - puppet.hiera_config_path = 'hiera.yaml' - end - - ## Custom Manifest: ensure vagrant-mounted event - # - # Note: future parser allow heredoc syntax in the puppet manifest (since puppet 3.5) - config.vm.provision 'puppet' do |puppet| - puppet.environment_path = 'puppet/environment' - puppet.environment = environment - puppet.manifests_path = "puppet/environment/#{environment}/manifests" - puppet.module_path = [ - "puppet/environment/#{environment}/modules_contrib", - "puppet/environment/#{environment}/modules", - ] - puppet.manifest_file = 'vagrant_mounted.pp' - puppet.hiera_config_path = 'hiera.yaml' - end - - ## Custom Manifest: install redis client / server - # - # Note: future parser allow heredoc syntax in the puppet manifest (since puppet 3.5) - config.vm.provision 'puppet' do |puppet| - puppet.environment_path = 'puppet/environment' - puppet.environment = environment - puppet.manifests_path = "puppet/environment/#{environment}/manifests" - puppet.module_path = [ - "puppet/environment/#{environment}/modules_contrib", - "puppet/environment/#{environment}/modules", - ] - puppet.manifest_file = 'configure_redis.pp' - puppet.hiera_config_path = 'hiera.yaml' - end - - ## Custom Manifest: configure system (i.e. system timezone) - config.vm.provision 'puppet' do |puppet| - puppet.environment_path = 'puppet/environment' - puppet.environment = environment - puppet.manifests_path = "puppet/environment/#{environment}/manifests" - puppet.module_path = [ - "puppet/environment/#{environment}/modules_contrib", - "puppet/environment/#{environment}/modules", - ] - puppet.manifest_file = 'configure_system.pp' - end - - ## Custom Manifest: define webcompilers - # - # Note: future parser allow heredoc sytnax (since puppet 3.5), and allows - # array iteration in the puppet manifest. - config.vm.provision 'puppet' do |puppet| - puppet.environment_path = 'puppet/environment' - puppet.environment = environment - puppet.manifests_path = "puppet/environment/#{environment}/manifests" - puppet.module_path = [ - "puppet/environment/#{environment}/modules_contrib", - "puppet/environment/#{environment}/modules", - ] - puppet.manifest_file = 'compile_asset.pp' - puppet.hiera_config_path = 'hiera.yaml' - end - - ## Custom Manifest: install, and configure SQL database - config.vm.provision 'puppet' do |puppet| - puppet.environment_path = 'puppet/environment' - puppet.environment = environment - puppet.manifests_path = "puppet/environment/#{environment}/manifests" - puppet.module_path = [ - "puppet/environment/#{environment}/modules_contrib", - "puppet/environment/#{environment}/modules", - ] - puppet.manifest_file = 'setup_database.pp' - puppet.hiera_config_path = 'hiera.yaml' - end - - ## Custom Manifest: start webserver - # - # Note: future parser allow heredoc syntax in the puppet manifest (since puppet 3.5) - config.vm.provision 'puppet' do |puppet| - puppet.environment_path = 'puppet/environment' - puppet.environment = environment - puppet.manifests_path = "puppet/environment/#{environment}/manifests" - puppet.module_path = [ - "puppet/environment/#{environment}/modules_contrib", - "puppet/environment/#{environment}/modules", - ] - puppet.manifest_file = 'start_webserver.pp' - puppet.hiera_config_path = 'hiera.yaml' - end - - # clean up files on the host after 'vagrant destroy' + ## clean up files on the host after 'vagrant destroy' config.trigger.after :destroy do run 'rm -Rf log/database' run 'rm -Rf log/application' diff --git a/database.dockerfile b/database.dockerfile index 5ff6527f8..aea2d2fdd 100644 --- a/database.dockerfile +++ b/database.dockerfile @@ -6,7 +6,7 @@ ENV ENVIRONMENT docker ENV ENVIRONMENT_DIR $ROOT_PROJECT/puppet/environment/$ENVIRONMENT ## provision with puppet -RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/manifests/setup_database.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test +RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/modules/database/manifests/init.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test ## executed everytime container starts -CMD ["mysqld"] \ No newline at end of file +CMD ["/bin/sh", "-c", "mysqld"] \ No newline at end of file diff --git a/default.dockerfile b/default.dockerfile index 31cb7e79b..728e0b2f1 100644 --- a/default.dockerfile +++ b/default.dockerfile @@ -10,8 +10,9 @@ RUN mkdir /var/machine-learning COPY . /var/machine-learning ## install git, wget, pip -# -# Note: r10k requires 'git' installed +## +## Note: r10k requires 'git' installed +## RUN apt-get -y update RUN apt-get -y install git=1:1.9.1-1ubuntu0.3 RUN apt-get -y install wget=1.15-1ubuntu1.14.04.2 @@ -25,7 +26,7 @@ RUN apt-get -y install puppet-agent ## install r10k RUN apt-get -y install rubygems-integration=1.5 -RUN gem install r10k -v 2.2.0 +RUN gem install r10k -v 2.5.2 ## install pytest-cov RUN pip install pytest-cov==2.4.0 @@ -35,12 +36,7 @@ RUN mkdir -p $ENVIRONMENT_DIR/modules_contrib/ RUN PUPPETFILE=$ENVIRONMENT_DIR/Puppetfile PUPPETFILE_DIR=$ENVIRONMENT_DIR/modules_contrib/ r10k puppetfile install ## provision with puppet -RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/manifests/install_packages.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test -RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/manifests/install_sklearn.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test -RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/manifests/configure_system.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test -RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/manifests/compile_asset.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test --debug - -## show directory -RUN ls -l $ROOT_PROJECT/interface/static/js -RUN ls -l $ROOT_PROJECT/interface/static/css -RUN ls -l $ROOT_PROJECT/interface/static/img +RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/modules/package/manifests/init.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test +RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/modules/sklearn/manifests/init.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test +RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/modules/system/manifests/init.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test +RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/modules/compiler/manifests/init.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test --debug diff --git a/hiera/test/hiera.yaml b/hiera/test/hiera.yaml index 17cfa127a..938dd9ec3 100644 --- a/hiera/test/hiera.yaml +++ b/hiera/test/hiera.yaml @@ -1,10 +1,12 @@ --- -:backends: - - yaml +version: 5 +defaults: + datadir: /var/machine-learning/hiera + data_hash: yaml_data -:yaml: - :datadir: /var/machine-learning/hiera +hierarchy: + - name: 'General build packages' + path: packages.yaml -:hierarchy: - - packages - - test/hiera/settings \ No newline at end of file + - name: 'General application settings' + path: test/hiera/settings.yaml \ No newline at end of file diff --git a/puppet/environment/docker/Puppetfile b/puppet/environment/docker/Puppetfile index a0aecaaaa..5ac7f378a 100644 --- a/puppet/environment/docker/Puppetfile +++ b/puppet/environment/docker/Puppetfile @@ -5,7 +5,7 @@ ## Install Module: stdlib (apt dependency) mod 'stdlib', :git => 'https://github.com/puppetlabs/puppetlabs-stdlib.git', - :ref => '4.11.0' + :ref => '4.15.0' ## Install Module: apt (from master) mod 'apt', @@ -14,8 +14,8 @@ mod 'apt', ## Install Module: nodejs mod 'nodejs', - :git => 'https://github.com/puppet-community/puppet-nodejs.git', - :ref => 'v1.3.0' + :git => 'https://github.com/voxpupuli/puppet-nodejs.git', + :ref => 'v2.3.0' ## Install Module: git mod 'git', diff --git a/puppet/environment/docker/manifests/compile_asset.pp b/puppet/environment/docker/manifests/compile_asset.pp deleted file mode 100644 index 31a12fcef..000000000 --- a/puppet/environment/docker/manifests/compile_asset.pp +++ /dev/null @@ -1,47 +0,0 @@ -### -### compile_asset.pp: install, configure, and run initial compile against -### source files. -### - -## ensure log directory, package dependencies -class dependencies { - require system::log_directory - contain package::webcompilers -} - -## configure webcompilers -class configure { - ## set dependency - require dependencies - - ## configure webcompilers - require system::webcompiler_directory - contain compiler::webcompilers -} - -## start webcompilers -class start { - ## set dependency - require dependencies - require configure - - ## start compiler(s) - contain compiler::start_sass - contain compiler::start_uglifyjs - contain compiler::start_browserify - contain compiler::start_imagemin -} - -## initial compile -class initiate { - ## set dependency - require dependencies - require configure - require start - - ## initial compile - contain compiler::initial_compile -} - -## initiate -include initiate \ No newline at end of file diff --git a/puppet/environment/docker/manifests/configure_redis.pp b/puppet/environment/docker/manifests/configure_redis.pp deleted file mode 100644 index 2d30d2f8e..000000000 --- a/puppet/environment/docker/manifests/configure_redis.pp +++ /dev/null @@ -1,9 +0,0 @@ -### -### configure_redis.pp: install redis client, and redis server. -### - -## install redis-server -include package::redis_server - -## configure redis-server -include redis::configuration diff --git a/puppet/environment/docker/manifests/configure_system.pp b/puppet/environment/docker/manifests/configure_system.pp deleted file mode 100644 index 6994f9617..000000000 --- a/puppet/environment/docker/manifests/configure_system.pp +++ /dev/null @@ -1,9 +0,0 @@ -### -### configure_system.pp: configure system with general requirements. -### - -## ensure log directory -require system::log_directory - -## define system timezone -include system::set_timezone \ No newline at end of file diff --git a/puppet/environment/docker/manifests/install_packages.pp b/puppet/environment/docker/manifests/install_packages.pp deleted file mode 100644 index ca70d0d33..000000000 --- a/puppet/environment/docker/manifests/install_packages.pp +++ /dev/null @@ -1,42 +0,0 @@ -### -### install_packages.pp: install general packages. -### - -## nodejs, with npm: this cannot be wrapped into a module, and included, as -## needed. Puppet will only allow one instance of this class, regardless of -## of its implementation. -class install_nodejs { - ## set dependency - require apt - - ## install nodejs, with npm - class { 'nodejs': - repo_url_suffix => '5.x', - } - contain nodejs -} - -## general packages -class general_packages { - ## set dependency - require apt - require install_nodejs - require system::webcompiler_directory - - ## install packages - contain package::inotify_tools - contain package::react_presets - contain package::jsonschema - contain package::xmltodict - contain package::six - contain package::fetch - contain package::pyyaml - contain package::flask_script - contain package::pytest_flask - contain package::python_dev - contain package::libssl_dev - contain package::scrypt -} - -## initiate -include general_packages diff --git a/puppet/environment/docker/manifests/install_sklearn.pp b/puppet/environment/docker/manifests/install_sklearn.pp deleted file mode 100644 index 750a8bc7f..000000000 --- a/puppet/environment/docker/manifests/install_sklearn.pp +++ /dev/null @@ -1,18 +0,0 @@ -### -### install_sklearn.pp: install sklearn, with all necessary dependencies. -### - -## create '/vagrant/build/' directory -include system::build_directory - -## install sklearn dependencies -include package::sklearn_dependencies - -## download scikit-learn -include package::sklearn - -## build scikit-learn -include sklearn::build_sklearn - -## install scikit-learn -include sklearn::install_sklearn \ No newline at end of file diff --git a/puppet/environment/docker/manifests/setup_database.pp b/puppet/environment/docker/manifests/setup_database.pp deleted file mode 100644 index 18fee0b6a..000000000 --- a/puppet/environment/docker/manifests/setup_database.pp +++ /dev/null @@ -1,24 +0,0 @@ -### -### setup_database.pp: install client, and initialize database tables. -### - -## install sql -class install_sql { - ## install mariadb - contain database::server - - ## install mariadb client - contain database::client - - ## install mariad bindings - contain database::bindings -} - -## create database tables -class create_db { - require install_sql - contain database::database -} - -## initiate -include create_db diff --git a/puppet/environment/docker/manifests/site.pp b/puppet/environment/docker/manifests/site.pp new file mode 100644 index 000000000..181cde8fa --- /dev/null +++ b/puppet/environment/docker/manifests/site.pp @@ -0,0 +1,14 @@ +### +### site.pp: load all necessary modules. +### +### Note: this file is not currently implemented, since the docker environment +### employs dockerfiles, where corresponding 'init.pp' are containd. +### + +contain package +contain sklearn +contain redis +contain system +contain compiler +contain database +contain webserver \ No newline at end of file diff --git a/puppet/environment/docker/manifests/start_webserver.pp b/puppet/environment/docker/manifests/start_webserver.pp deleted file mode 100644 index 30800cc05..000000000 --- a/puppet/environment/docker/manifests/start_webserver.pp +++ /dev/null @@ -1,15 +0,0 @@ -### -### start_webserver.pp: start webserver, and ensure all client services exist, -### and properly configured. -### - -## install mariadb -include database::client -include database::bindings - -## install redis client -include package::redis_client - -## install webserver -include package::gunicorn -include webserver::service \ No newline at end of file diff --git a/puppet/environment/docker/modules/compiler/manifests/init.pp b/puppet/environment/docker/modules/compiler/manifests/init.pp new file mode 100644 index 000000000..b98efa3c4 --- /dev/null +++ b/puppet/environment/docker/modules/compiler/manifests/init.pp @@ -0,0 +1,22 @@ +### +### init.pp: install, configure, and run initial compile against source files. +### + +class compiler { + ## ensure log directory, package dependencies + require system::log_directory + + ## configure webcompilers + require system::webcompiler_directory + contain compiler::webcompilers + + ## start compiler(s) + contain compiler::start_sass + contain compiler::start_uglifyjs + contain compiler::start_browserify + contain compiler::start_imagemin + + ## initial compile + contain compiler::initial_compile +} +contain compiler \ No newline at end of file diff --git a/puppet/environment/docker/modules/compiler/manifests/initial_compile.pp b/puppet/environment/docker/modules/compiler/manifests/initial_compile.pp index f00869789..e3952e5ec 100644 --- a/puppet/environment/docker/modules/compiler/manifests/initial_compile.pp +++ b/puppet/environment/docker/modules/compiler/manifests/initial_compile.pp @@ -2,6 +2,8 @@ ### initial_compile.pp, manually compile javascript. ### class compiler::initial_compile { + include package::webcompilers + ## local variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -21,5 +23,6 @@ cwd => "${dev_env_path}/modules/compiler/scripts", path => '/usr/bin', provider => shell, + require => Class['package::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/compiler/manifests/start_browserify.pp b/puppet/environment/docker/modules/compiler/manifests/start_browserify.pp index 2d681b620..a64b1b9ef 100644 --- a/puppet/environment/docker/modules/compiler/manifests/start_browserify.pp +++ b/puppet/environment/docker/modules/compiler/manifests/start_browserify.pp @@ -2,6 +2,8 @@ ### start_browserify.pp, compile jsx into javascript. ### class compiler::start_browserify { + include compiler::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -14,5 +16,6 @@ cwd => "${dev_env_path}/modules/compiler/scripts", path => '/usr/bin', provider => shell, + require => Class['compiler::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/compiler/manifests/start_imagemin.pp b/puppet/environment/docker/modules/compiler/manifests/start_imagemin.pp index 73e8579d4..fce1271b7 100644 --- a/puppet/environment/docker/modules/compiler/manifests/start_imagemin.pp +++ b/puppet/environment/docker/modules/compiler/manifests/start_imagemin.pp @@ -2,6 +2,8 @@ ### start_imagemin.pp, minify images, when possible. ### class compiler::start_imagemin { + include compiler::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -13,5 +15,6 @@ command => "./imagemin ${root_dir}", cwd => "${dev_env_path}/modules/compiler/scripts", provider => shell, + require => Class['compiler::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/compiler/manifests/start_sass.pp b/puppet/environment/docker/modules/compiler/manifests/start_sass.pp index e1ef8e7f9..30d6e1f0f 100644 --- a/puppet/environment/docker/modules/compiler/manifests/start_sass.pp +++ b/puppet/environment/docker/modules/compiler/manifests/start_sass.pp @@ -2,6 +2,8 @@ ### start_sass.pp, minify scss into css. ### class compiler::start_sass { + include compiler::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -14,5 +16,6 @@ cwd => "${dev_env_path}/modules/compiler/scripts", timeout => 700, provider => shell, + require => Class['compiler::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/compiler/manifests/start_uglifyjs.pp b/puppet/environment/docker/modules/compiler/manifests/start_uglifyjs.pp index b54ed769a..8c1548887 100644 --- a/puppet/environment/docker/modules/compiler/manifests/start_uglifyjs.pp +++ b/puppet/environment/docker/modules/compiler/manifests/start_uglifyjs.pp @@ -2,6 +2,8 @@ ### start_uglifyjs.pp, minify javascript. ### class compiler::start_uglifyjs { + include compiler::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -13,5 +15,6 @@ command => "./uglifyjs ${root_dir}", cwd => "${dev_env_path}/modules/compiler/scripts", provider => shell, + require => Class['compiler::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/compiler/manifests/webcompilers.pp b/puppet/environment/docker/modules/compiler/manifests/webcompilers.pp index 43c603b46..e8ba53611 100644 --- a/puppet/environment/docker/modules/compiler/manifests/webcompilers.pp +++ b/puppet/environment/docker/modules/compiler/manifests/webcompilers.pp @@ -2,6 +2,9 @@ ### webcompilers.pp, create webcompiler services. ### class compiler::webcompilers { + include package::react_presets + include package::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -34,6 +37,10 @@ file { "/etc/init/${compiler}.conf": ensure => file, content => dos2unix(template($template_path)), + require => [ + Class['package::webcompilers'], + Class['package::react_presets'], + ], } ## dos2unix upstart: convert clrf (windows to linux) in case host @@ -42,6 +49,10 @@ ensure => file, content => dos2unix(template("${compiler_dir}/${compiler}")), mode => '0755', + require => [ + Class['package::webcompilers'], + Class['package::react_presets'], + ], } } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/database/manifests/database.pp b/puppet/environment/docker/modules/database/manifests/database.pp index 7c98934a5..80b31d999 100644 --- a/puppet/environment/docker/modules/database/manifests/database.pp +++ b/puppet/environment/docker/modules/database/manifests/database.pp @@ -2,6 +2,8 @@ ### database.pp, create mariadb database structure. ### class database::database { + include package::pyyaml + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -19,5 +21,6 @@ command => "python setup_tables.py ${root_dir} ${vagrant_mounted}", cwd => $script_dir, path => '/usr/bin', + require => Class['package::pyyaml'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/manifests/setup_database.pp b/puppet/environment/docker/modules/database/manifests/init.pp similarity index 51% rename from puppet/environment/vagrant/manifests/setup_database.pp rename to puppet/environment/docker/modules/database/manifests/init.pp index 18fee0b6a..3db98127a 100644 --- a/puppet/environment/vagrant/manifests/setup_database.pp +++ b/puppet/environment/docker/modules/database/manifests/init.pp @@ -1,9 +1,8 @@ ### -### setup_database.pp: install client, and initialize database tables. +### init.pp: install client, and initialize database tables. ### -## install sql -class install_sql { +class database { ## install mariadb contain database::server @@ -12,13 +11,8 @@ ## install mariad bindings contain database::bindings -} -## create database tables -class create_db { - require install_sql + ## create database tables contain database::database } - -## initiate -include create_db +contain database \ No newline at end of file diff --git a/puppet/environment/docker/modules/package/manifests/fetch.pp b/puppet/environment/docker/modules/package/manifests/fetch.pp index 41f83ece6..5ecb0556f 100644 --- a/puppet/environment/docker/modules/package/manifests/fetch.pp +++ b/puppet/environment/docker/modules/package/manifests/fetch.pp @@ -2,6 +2,8 @@ ### fetch.pp, install package. ### class package::fetch { + include package::nodejs + ## local variables $hiera_dev = lookup('development') $version = $hiera_dev['npm']['fetch'] @@ -10,5 +12,6 @@ package { "whatwg-fetch@${version}": ensure => 'present', provider => 'npm', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/package/manifests/init.pp b/puppet/environment/docker/modules/package/manifests/init.pp new file mode 100644 index 000000000..ae1c5ae9e --- /dev/null +++ b/puppet/environment/docker/modules/package/manifests/init.pp @@ -0,0 +1,22 @@ +### +### init.pp: install general packages. +### + +class package { + require apt + + ## general packages + contain package::nodejs + contain package::inotify_tools + contain package::react_presets + contain package::jsonschema + contain package::xmltodict + contain package::six + contain package::fetch + contain package::pyyaml + contain package::flask_script + contain package::pytest_flask + contain package::libssl_dev + contain package::scrypt +} +contain package \ No newline at end of file diff --git a/puppet/environment/docker/modules/package/manifests/inotify_tools.pp b/puppet/environment/docker/modules/package/manifests/inotify_tools.pp index 58fa7d7d2..c93b07d32 100644 --- a/puppet/environment/docker/modules/package/manifests/inotify_tools.pp +++ b/puppet/environment/docker/modules/package/manifests/inotify_tools.pp @@ -2,6 +2,8 @@ ### inotify_tools.pp, install package. ### class package::inotify_tools { + include package::nodejs + ## update apt-get require apt @@ -10,6 +12,7 @@ $version = $hiera_dev['apt']['inotify-tools'] package { "inotify-tools=${version}": - ensure => 'installed', + ensure => 'installed', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/package/manifests/nodejs.pp b/puppet/environment/docker/modules/package/manifests/nodejs.pp new file mode 100644 index 000000000..f8e32f1c6 --- /dev/null +++ b/puppet/environment/docker/modules/package/manifests/nodejs.pp @@ -0,0 +1,13 @@ +### +### nodejs.pp: install nodejs, and npm. +### + +class package::nodejs { + ## install nodejs, with npm + class { 'nodejs': + manage_package_repo => true, + repo_url_suffix => '4.x', + nodejs_package_ensure => 'latest' + } + contain nodejs +} diff --git a/puppet/environment/docker/modules/package/manifests/react_presets.pp b/puppet/environment/docker/modules/package/manifests/react_presets.pp index ae86a4678..641f10715 100644 --- a/puppet/environment/docker/modules/package/manifests/react_presets.pp +++ b/puppet/environment/docker/modules/package/manifests/react_presets.pp @@ -2,6 +2,8 @@ ### react_presets.pp, install necessary webpackages defined in 'package.json'. ### class package::react_presets { + include package::nodejs + ## local variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -11,5 +13,6 @@ command => 'npm install --no-bin-links', cwd => "${root_dir}/src/jsx/", path => '/usr/bin', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/package/manifests/scrypt.pp b/puppet/environment/docker/modules/package/manifests/scrypt.pp index ca68e4163..0ef266ae7 100644 --- a/puppet/environment/docker/modules/package/manifests/scrypt.pp +++ b/puppet/environment/docker/modules/package/manifests/scrypt.pp @@ -3,6 +3,7 @@ ### class package::scrypt { require python + include package::python_dev ## local variables $hiera_dev = lookup('development') @@ -11,5 +12,6 @@ package { 'scrypt': ensure => $version, provider => 'pip', + require => Class['package::python_dev'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/package/manifests/sklearn_dependencies.pp b/puppet/environment/docker/modules/package/manifests/sklearn_dependencies.pp index 2e9802c31..15c7f3e2a 100644 --- a/puppet/environment/docker/modules/package/manifests/sklearn_dependencies.pp +++ b/puppet/environment/docker/modules/package/manifests/sklearn_dependencies.pp @@ -2,9 +2,10 @@ ### sklearn_dependencies.pp, install sklearn related packages. ### class package::sklearn_dependencies { + contain package::python_dev + ## local variables $hiera_dev = lookup('development') - $version_python_dev = $hiera_dev['apt']['python-dev'] $version_python_numpy = $hiera_dev['apt']['python-numpy'] $version_python_scipy = $hiera_dev['apt']['python-scipy'] $version_libatlas_dev = $hiera_dev['apt']['libatlas-dev'] @@ -13,7 +14,6 @@ $version_ipython = $hiera_dev['apt']['ipython'] $dependencies = [ - "python-dev=${version_python_dev}", "python-numpy=${$version_python_numpy}", "python-scipy=${version_python_scipy}", "libatlas-dev=${version_libatlas_dev}", diff --git a/puppet/environment/docker/modules/package/manifests/webcompilers.pp b/puppet/environment/docker/modules/package/manifests/webcompilers.pp index 40cdfb8f1..263632a37 100644 --- a/puppet/environment/docker/modules/package/manifests/webcompilers.pp +++ b/puppet/environment/docker/modules/package/manifests/webcompilers.pp @@ -2,6 +2,8 @@ ### webcompilers.pp, install webcompiler packages. ### class package::webcompilers { + include package::nodejs + ## hiera attributes $hiera_dev = lookup('development') $version_uglify_js = $hiera_dev['npm']['uglify-js'] @@ -25,5 +27,6 @@ package { $webcompilers: ensure => 'present', provider => 'npm', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/package/manifests/xmltodict.pp b/puppet/environment/docker/modules/package/manifests/xmltodict.pp index 5c1d3ee47..768fdf767 100644 --- a/puppet/environment/docker/modules/package/manifests/xmltodict.pp +++ b/puppet/environment/docker/modules/package/manifests/xmltodict.pp @@ -3,6 +3,7 @@ ### class package::xmltodict { include python + include package::nodejs ## local variables $hiera_dev = lookup('development') @@ -11,5 +12,6 @@ package { 'xmltodict': ensure => $version, provider => 'pip', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/redis/manifests/init.pp b/puppet/environment/docker/modules/redis/manifests/init.pp new file mode 100644 index 000000000..8a73090f4 --- /dev/null +++ b/puppet/environment/docker/modules/redis/manifests/init.pp @@ -0,0 +1,9 @@ +### +### init.pp: install redis client, and redis server. +### + +class redis { + contain package::redis_server + contain redis::configuration +} +contain redis diff --git a/puppet/environment/docker/modules/sklearn/manifests/init.pp b/puppet/environment/docker/modules/sklearn/manifests/init.pp new file mode 100644 index 000000000..ad16997b0 --- /dev/null +++ b/puppet/environment/docker/modules/sklearn/manifests/init.pp @@ -0,0 +1,21 @@ +### +### init.pp: install sklearn, with all necessary dependencies. +### + +class sklearn { + ## create '/vagrant/build/' directory + contain system::build_directory + + ## install sklearn dependencies + contain package::sklearn_dependencies + + ## download scikit-learn + contain package::sklearn + + ## build scikit-learn + contain sklearn::build_sklearn + + ## install scikit-learn + contain sklearn::install_sklearn +} +contain sklearn \ No newline at end of file diff --git a/puppet/environment/docker/modules/system/manifests/init.pp b/puppet/environment/docker/modules/system/manifests/init.pp new file mode 100644 index 000000000..997a003de --- /dev/null +++ b/puppet/environment/docker/modules/system/manifests/init.pp @@ -0,0 +1,12 @@ +### +### init.pp: configure system with general requirements. +### + +class system { + ## ensure log directory + require system::log_directory + + ## define system timezone + contain system::set_timezone +} +contain system \ No newline at end of file diff --git a/puppet/environment/docker/modules/webserver/manifests/init.pp b/puppet/environment/docker/modules/webserver/manifests/init.pp new file mode 100644 index 000000000..085249444 --- /dev/null +++ b/puppet/environment/docker/modules/webserver/manifests/init.pp @@ -0,0 +1,18 @@ +### +### init.pp: start webserver, and ensure all client services exist, and +### properly configured. +### + +class webserver { + ## install mariadb + contain database::client + contain database::bindings + + ## install redis client + contain package::redis_client + + ## install webserver + contain package::gunicorn + contain webserver::service +} +contain webserver \ No newline at end of file diff --git a/puppet/environment/docker/modules/webserver/manifests/service.pp b/puppet/environment/docker/modules/webserver/manifests/service.pp index e08cac228..db47ce52d 100644 --- a/puppet/environment/docker/modules/webserver/manifests/service.pp +++ b/puppet/environment/docker/modules/webserver/manifests/service.pp @@ -2,6 +2,9 @@ ### service.pp, configure webserver(s), and corresponding proxy. ### class webserver::service { + include compiler::initial_compile + include webserver::start + ## variables $hiera_general = lookup('general') $hiera_development = lookup('development') @@ -24,10 +27,10 @@ $nginx_version = $hiera_development['apt']['nginx'] $nginx_proxy = "${nginx_reverse_proxy['proxy']}:${gunicorn_port}" - ## include webserver dependencies - include python - include python::flask - include python::requests + ## contain webserver dependencies + contain python + contain python::flask + contain python::requests ## nginx: installation class { 'nginx': @@ -42,8 +45,14 @@ ## dos2unix: convert clrf (windows to linux) in case host machine is ## windows. + ## + ## Note: when the application starts, particular package dependencies are + ## required to be installed, so the flask application can run. + ## file { '/etc/init/start_gunicorn.conf': ensure => file, content => dos2unix(template($template_path)), + require => Class['compiler::initial_compile'], + notify => Class['webserver::start'], } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/webserver/manifests/start.pp b/puppet/environment/docker/modules/webserver/manifests/start.pp index 24ac0eed6..f5354a85c 100644 --- a/puppet/environment/docker/modules/webserver/manifests/start.pp +++ b/puppet/environment/docker/modules/webserver/manifests/start.pp @@ -2,22 +2,26 @@ ### start.pp, ensure gunicorn webserver workers running. ### class webserver::start { - # variables + include webserver::service + + ## variables $hiera_general = lookup('general') $vagrant_mounted = $hiera_general['vagrant_implement'] - # run gunicorn + ## run gunicorn if $vagrant_mounted { # ensure service starts at boot service { 'start_gunicorn': - ensure => 'running', - enable => true, + ensure => 'running', + enable => true, + require => Class['webserver::service'], } } else { - # run and restart when needed + ## run and restart when needed service { 'start_gunicorn': - ensure => 'running', + ensure => 'running', + require => Class['webserver::service'], } } } \ No newline at end of file diff --git a/puppet/environment/vagrant/Puppetfile b/puppet/environment/vagrant/Puppetfile index dbfe6b4f6..0c9f6375a 100644 --- a/puppet/environment/vagrant/Puppetfile +++ b/puppet/environment/vagrant/Puppetfile @@ -8,7 +8,7 @@ ## Install Module: stdlib (apt dependency) mod 'stdlib', :git => 'git@github.com:puppetlabs/puppetlabs-stdlib.git', - :ref => '4.11.0' + :ref => '4.15.0' ## Install Module: apt (from master) mod 'apt', @@ -17,8 +17,8 @@ mod 'apt', ## Install Module: nodejs mod 'nodejs', - :git => 'git@github.com:puppet-community/puppet-nodejs.git', - :ref => 'v1.3.0' + :git => 'git@github.com:voxpupuli/puppet-nodejs.git', + :ref => 'v2.3.0' ## Install Module: git mod 'git', diff --git a/puppet/environment/vagrant/manifests/compile_asset.pp b/puppet/environment/vagrant/manifests/compile_asset.pp deleted file mode 100644 index 31a12fcef..000000000 --- a/puppet/environment/vagrant/manifests/compile_asset.pp +++ /dev/null @@ -1,47 +0,0 @@ -### -### compile_asset.pp: install, configure, and run initial compile against -### source files. -### - -## ensure log directory, package dependencies -class dependencies { - require system::log_directory - contain package::webcompilers -} - -## configure webcompilers -class configure { - ## set dependency - require dependencies - - ## configure webcompilers - require system::webcompiler_directory - contain compiler::webcompilers -} - -## start webcompilers -class start { - ## set dependency - require dependencies - require configure - - ## start compiler(s) - contain compiler::start_sass - contain compiler::start_uglifyjs - contain compiler::start_browserify - contain compiler::start_imagemin -} - -## initial compile -class initiate { - ## set dependency - require dependencies - require configure - require start - - ## initial compile - contain compiler::initial_compile -} - -## initiate -include initiate \ No newline at end of file diff --git a/puppet/environment/vagrant/manifests/configure_redis.pp b/puppet/environment/vagrant/manifests/configure_redis.pp deleted file mode 100644 index 0afc6f63a..000000000 --- a/puppet/environment/vagrant/manifests/configure_redis.pp +++ /dev/null @@ -1,9 +0,0 @@ -### -### configure_redis.pp: install redis client, and redis server. -### - -## install redis client -include package::redis_client - -## install redis-server -include package::redis_server \ No newline at end of file diff --git a/puppet/environment/vagrant/manifests/configure_system.pp b/puppet/environment/vagrant/manifests/configure_system.pp deleted file mode 100644 index 27121b1db..000000000 --- a/puppet/environment/vagrant/manifests/configure_system.pp +++ /dev/null @@ -1,6 +0,0 @@ -### -### configure_system.pp: configure system with general requirements. -### - -## define system timezone -include system::set_timezone \ No newline at end of file diff --git a/puppet/environment/vagrant/manifests/install_packages.pp b/puppet/environment/vagrant/manifests/install_packages.pp deleted file mode 100644 index ca70d0d33..000000000 --- a/puppet/environment/vagrant/manifests/install_packages.pp +++ /dev/null @@ -1,42 +0,0 @@ -### -### install_packages.pp: install general packages. -### - -## nodejs, with npm: this cannot be wrapped into a module, and included, as -## needed. Puppet will only allow one instance of this class, regardless of -## of its implementation. -class install_nodejs { - ## set dependency - require apt - - ## install nodejs, with npm - class { 'nodejs': - repo_url_suffix => '5.x', - } - contain nodejs -} - -## general packages -class general_packages { - ## set dependency - require apt - require install_nodejs - require system::webcompiler_directory - - ## install packages - contain package::inotify_tools - contain package::react_presets - contain package::jsonschema - contain package::xmltodict - contain package::six - contain package::fetch - contain package::pyyaml - contain package::flask_script - contain package::pytest_flask - contain package::python_dev - contain package::libssl_dev - contain package::scrypt -} - -## initiate -include general_packages diff --git a/puppet/environment/vagrant/manifests/install_sklearn.pp b/puppet/environment/vagrant/manifests/install_sklearn.pp deleted file mode 100644 index 750a8bc7f..000000000 --- a/puppet/environment/vagrant/manifests/install_sklearn.pp +++ /dev/null @@ -1,18 +0,0 @@ -### -### install_sklearn.pp: install sklearn, with all necessary dependencies. -### - -## create '/vagrant/build/' directory -include system::build_directory - -## install sklearn dependencies -include package::sklearn_dependencies - -## download scikit-learn -include package::sklearn - -## build scikit-learn -include sklearn::build_sklearn - -## install scikit-learn -include sklearn::install_sklearn \ No newline at end of file diff --git a/puppet/environment/vagrant/manifests/site.pp b/puppet/environment/vagrant/manifests/site.pp new file mode 100644 index 000000000..567444f1c --- /dev/null +++ b/puppet/environment/vagrant/manifests/site.pp @@ -0,0 +1,12 @@ +### +### site.pp: load all necessary modules. +### + +contain package +contain sklearn +contain vagrant +contain redis +contain system +contain compiler +contain database +contain webserver \ No newline at end of file diff --git a/puppet/environment/vagrant/manifests/start_webserver.pp b/puppet/environment/vagrant/manifests/start_webserver.pp deleted file mode 100644 index d5b703d70..000000000 --- a/puppet/environment/vagrant/manifests/start_webserver.pp +++ /dev/null @@ -1,13 +0,0 @@ -### -### start_webserver.pp: start webserver. -### - -## ensure log directory -require system::log_directory - -## install webserver -include webserver::service - -## start webservers -include package::gunicorn -include webserver::start \ No newline at end of file diff --git a/puppet/environment/vagrant/manifests/vagrant_mounted.pp b/puppet/environment/vagrant/manifests/vagrant_mounted.pp deleted file mode 100644 index 85fef01f2..000000000 --- a/puppet/environment/vagrant/manifests/vagrant_mounted.pp +++ /dev/null @@ -1,11 +0,0 @@ -### -### vagrant_mounted.pp: ensure 'vagrant-mounted' event fires, when '/vagrant' -### shared directory, is mounted within the guest virtual -### machine. -### - -## configure service -include vagrant::service - -## start service -include vagrant::start \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/compiler/manifests/init.pp b/puppet/environment/vagrant/modules/compiler/manifests/init.pp new file mode 100644 index 000000000..b1bf78361 --- /dev/null +++ b/puppet/environment/vagrant/modules/compiler/manifests/init.pp @@ -0,0 +1,21 @@ +### +### init.pp: install, configure, and run initial compile against source files. +### + +class compiler { + ## ensure log directory, package dependencies + require system::log_directory + + ## configure webcompilers + require system::webcompiler_directory + contain compiler::webcompilers + + ## start compiler(s) + contain compiler::start_sass + contain compiler::start_uglifyjs + contain compiler::start_browserify + contain compiler::start_imagemin + + ## initial compile + contain compiler::initial_compile +} \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/compiler/manifests/initial_compile.pp b/puppet/environment/vagrant/modules/compiler/manifests/initial_compile.pp index 95b3ff3a8..41683010f 100644 --- a/puppet/environment/vagrant/modules/compiler/manifests/initial_compile.pp +++ b/puppet/environment/vagrant/modules/compiler/manifests/initial_compile.pp @@ -2,6 +2,8 @@ ### initial_compile.pp, manually compile javascript. ### class compiler::initial_compile { + include package::webcompilers + ## local variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -46,6 +48,7 @@ command => $touch_files, onlyif => $check_files, provider => shell, + require => Class['package::webcompilers'], } } @@ -54,5 +57,6 @@ command => "node-sass ${source} ${asset}", path => '/usr/bin', provider => shell, + require => Class['package::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/compiler/manifests/start_browserify.pp b/puppet/environment/vagrant/modules/compiler/manifests/start_browserify.pp index 0b591ce3c..505d94bb7 100644 --- a/puppet/environment/vagrant/modules/compiler/manifests/start_browserify.pp +++ b/puppet/environment/vagrant/modules/compiler/manifests/start_browserify.pp @@ -2,6 +2,8 @@ ### start_browserify.pp, compile jsx into javascript. ### class compiler::start_browserify { + include compiler::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -10,7 +12,8 @@ ## ensure service starts at boot service { 'browserify': - ensure => 'running', - enable => true, + ensure => 'running', + enable => true, + require => Class['compiler::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/compiler/manifests/start_imagemin.pp b/puppet/environment/vagrant/modules/compiler/manifests/start_imagemin.pp index 1fc0ae78c..5a858da78 100644 --- a/puppet/environment/vagrant/modules/compiler/manifests/start_imagemin.pp +++ b/puppet/environment/vagrant/modules/compiler/manifests/start_imagemin.pp @@ -2,6 +2,8 @@ ### start_imagemin.pp, minify images, when possible. ### class compiler::start_imagemin { + include compiler::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -10,7 +12,8 @@ ## ensure service starts at boot service { 'imagemin': - ensure => 'running', - enable => true, + ensure => 'running', + enable => true, + require => Class['compiler::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/compiler/manifests/start_sass.pp b/puppet/environment/vagrant/modules/compiler/manifests/start_sass.pp index 3671cad27..9cc3402f0 100644 --- a/puppet/environment/vagrant/modules/compiler/manifests/start_sass.pp +++ b/puppet/environment/vagrant/modules/compiler/manifests/start_sass.pp @@ -2,8 +2,11 @@ ### start_sass.pp, ensure custom sass service running. ### class compiler::start_sass { + include compiler::webcompilers + service { 'sass': - ensure => 'running', - enable => true, + ensure => 'running', + enable => true, + require => Class['compiler::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/compiler/manifests/start_uglifyjs.pp b/puppet/environment/vagrant/modules/compiler/manifests/start_uglifyjs.pp index 6d47153a4..f31ec556c 100644 --- a/puppet/environment/vagrant/modules/compiler/manifests/start_uglifyjs.pp +++ b/puppet/environment/vagrant/modules/compiler/manifests/start_uglifyjs.pp @@ -2,6 +2,8 @@ ### start_uglifyjs.pp, ensure custom uglifyjs service running. ### class compiler::start_uglifyjs { + include compiler::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -10,7 +12,8 @@ ## ensure service starts at boot service { 'uglifyjs': - ensure => 'running', - enable => true, + ensure => 'running', + enable => true, + require => Class['compiler::webcompilers'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/compiler/manifests/webcompilers.pp b/puppet/environment/vagrant/modules/compiler/manifests/webcompilers.pp index ef280278b..c90316f67 100644 --- a/puppet/environment/vagrant/modules/compiler/manifests/webcompilers.pp +++ b/puppet/environment/vagrant/modules/compiler/manifests/webcompilers.pp @@ -2,6 +2,9 @@ ### webcompilers.pp, create webcompiler services. ### class compiler::webcompilers { + include package::react_presets + include package::webcompilers + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -35,6 +38,10 @@ file { "/etc/init/${compiler}.conf": ensure => file, content => dos2unix(template($template_path)), + require => [ + Class['package::webcompilers'], + Class['package::react_presets'], + ], } ## dos2unix upstart: convert clrf (windows to linux) in case host @@ -43,6 +50,10 @@ ensure => file, content => dos2unix(template("${compiler_dir}/${compiler}")), mode => '0755', + require => [ + Class['package::webcompilers'], + Class['package::react_presets'], + ], } } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/database/manifests/database.pp b/puppet/environment/vagrant/modules/database/manifests/database.pp index 7c98934a5..80b31d999 100644 --- a/puppet/environment/vagrant/modules/database/manifests/database.pp +++ b/puppet/environment/vagrant/modules/database/manifests/database.pp @@ -2,6 +2,8 @@ ### database.pp, create mariadb database structure. ### class database::database { + include package::pyyaml + ## variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -19,5 +21,6 @@ command => "python setup_tables.py ${root_dir} ${vagrant_mounted}", cwd => $script_dir, path => '/usr/bin', + require => Class['package::pyyaml'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/database/manifests/init.pp b/puppet/environment/vagrant/modules/database/manifests/init.pp new file mode 100644 index 000000000..cc49fe607 --- /dev/null +++ b/puppet/environment/vagrant/modules/database/manifests/init.pp @@ -0,0 +1,10 @@ +### +### init.pp: install client, and initialize database tables. +### + +class database { + contain database::server + contain database::client + contain database::bindings + contain database::database +} \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/package/manifests/fetch.pp b/puppet/environment/vagrant/modules/package/manifests/fetch.pp index 41f83ece6..5ecb0556f 100644 --- a/puppet/environment/vagrant/modules/package/manifests/fetch.pp +++ b/puppet/environment/vagrant/modules/package/manifests/fetch.pp @@ -2,6 +2,8 @@ ### fetch.pp, install package. ### class package::fetch { + include package::nodejs + ## local variables $hiera_dev = lookup('development') $version = $hiera_dev['npm']['fetch'] @@ -10,5 +12,6 @@ package { "whatwg-fetch@${version}": ensure => 'present', provider => 'npm', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/package/manifests/init.pp b/puppet/environment/vagrant/modules/package/manifests/init.pp new file mode 100644 index 000000000..559fe018d --- /dev/null +++ b/puppet/environment/vagrant/modules/package/manifests/init.pp @@ -0,0 +1,21 @@ +### +### init.pp: install general packages. +### + +class package { + require apt + + ## general packages + contain package::nodejs + contain package::inotify_tools + contain package::react_presets + contain package::jsonschema + contain package::xmltodict + contain package::six + contain package::fetch + contain package::pyyaml + contain package::flask_script + contain package::pytest_flask + contain package::libssl_dev + contain package::scrypt +} diff --git a/puppet/environment/vagrant/modules/package/manifests/inotify_tools.pp b/puppet/environment/vagrant/modules/package/manifests/inotify_tools.pp index 58fa7d7d2..c93b07d32 100644 --- a/puppet/environment/vagrant/modules/package/manifests/inotify_tools.pp +++ b/puppet/environment/vagrant/modules/package/manifests/inotify_tools.pp @@ -2,6 +2,8 @@ ### inotify_tools.pp, install package. ### class package::inotify_tools { + include package::nodejs + ## update apt-get require apt @@ -10,6 +12,7 @@ $version = $hiera_dev['apt']['inotify-tools'] package { "inotify-tools=${version}": - ensure => 'installed', + ensure => 'installed', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/package/manifests/nodejs.pp b/puppet/environment/vagrant/modules/package/manifests/nodejs.pp new file mode 100644 index 000000000..1082ed823 --- /dev/null +++ b/puppet/environment/vagrant/modules/package/manifests/nodejs.pp @@ -0,0 +1,11 @@ +### +### nodejs.pp: install nodejs, and npm. +### + +class package::nodejs { + ## install nodejs, with npm + class { 'nodejs': + repo_url_suffix => '5.x', + } + contain nodejs +} diff --git a/puppet/environment/vagrant/modules/package/manifests/react_presets.pp b/puppet/environment/vagrant/modules/package/manifests/react_presets.pp index ae86a4678..641f10715 100644 --- a/puppet/environment/vagrant/modules/package/manifests/react_presets.pp +++ b/puppet/environment/vagrant/modules/package/manifests/react_presets.pp @@ -2,6 +2,8 @@ ### react_presets.pp, install necessary webpackages defined in 'package.json'. ### class package::react_presets { + include package::nodejs + ## local variables $hiera_general = lookup('general') $root_dir = $hiera_general['root'] @@ -11,5 +13,6 @@ command => 'npm install --no-bin-links', cwd => "${root_dir}/src/jsx/", path => '/usr/bin', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/package/manifests/scrypt.pp b/puppet/environment/vagrant/modules/package/manifests/scrypt.pp index 1f8e40952..27324389e 100644 --- a/puppet/environment/vagrant/modules/package/manifests/scrypt.pp +++ b/puppet/environment/vagrant/modules/package/manifests/scrypt.pp @@ -3,6 +3,7 @@ ### class package::scrypt { require python + include package::python_dev ## local variables $hiera_dev = lookup('development') @@ -11,5 +12,6 @@ package { 'scrypt': ensure => $version, provider => 'pip', + require => Class['package::python_dev'], } } diff --git a/puppet/environment/vagrant/modules/package/manifests/sklearn_dependencies.pp b/puppet/environment/vagrant/modules/package/manifests/sklearn_dependencies.pp index 2e9802c31..15c7f3e2a 100644 --- a/puppet/environment/vagrant/modules/package/manifests/sklearn_dependencies.pp +++ b/puppet/environment/vagrant/modules/package/manifests/sklearn_dependencies.pp @@ -2,9 +2,10 @@ ### sklearn_dependencies.pp, install sklearn related packages. ### class package::sklearn_dependencies { + contain package::python_dev + ## local variables $hiera_dev = lookup('development') - $version_python_dev = $hiera_dev['apt']['python-dev'] $version_python_numpy = $hiera_dev['apt']['python-numpy'] $version_python_scipy = $hiera_dev['apt']['python-scipy'] $version_libatlas_dev = $hiera_dev['apt']['libatlas-dev'] @@ -13,7 +14,6 @@ $version_ipython = $hiera_dev['apt']['ipython'] $dependencies = [ - "python-dev=${version_python_dev}", "python-numpy=${$version_python_numpy}", "python-scipy=${version_python_scipy}", "libatlas-dev=${version_libatlas_dev}", diff --git a/puppet/environment/vagrant/modules/package/manifests/webcompilers.pp b/puppet/environment/vagrant/modules/package/manifests/webcompilers.pp index 40cdfb8f1..263632a37 100644 --- a/puppet/environment/vagrant/modules/package/manifests/webcompilers.pp +++ b/puppet/environment/vagrant/modules/package/manifests/webcompilers.pp @@ -2,6 +2,8 @@ ### webcompilers.pp, install webcompiler packages. ### class package::webcompilers { + include package::nodejs + ## hiera attributes $hiera_dev = lookup('development') $version_uglify_js = $hiera_dev['npm']['uglify-js'] @@ -25,5 +27,6 @@ package { $webcompilers: ensure => 'present', provider => 'npm', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/package/manifests/xmltodict.pp b/puppet/environment/vagrant/modules/package/manifests/xmltodict.pp index 017a9121d..2d9768247 100644 --- a/puppet/environment/vagrant/modules/package/manifests/xmltodict.pp +++ b/puppet/environment/vagrant/modules/package/manifests/xmltodict.pp @@ -3,6 +3,7 @@ ## class package::xmltodict { include python + include package::nodejs ## local variables $hiera_dev = lookup('development') @@ -11,5 +12,6 @@ package { 'xmltodict': ensure => $version, provider => 'pip', + require => Class['package::nodejs'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/redis/manifests/init.pp b/puppet/environment/vagrant/modules/redis/manifests/init.pp new file mode 100644 index 000000000..01c846303 --- /dev/null +++ b/puppet/environment/vagrant/modules/redis/manifests/init.pp @@ -0,0 +1,11 @@ +### +### init.pp: install redis client, and redis server. +### + +class redis { + ## install redis client + contain package::redis_client + + ## install redis-server + contain package::redis_server +} \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/sklearn/manifests/init.pp b/puppet/environment/vagrant/modules/sklearn/manifests/init.pp new file mode 100644 index 000000000..0d4b8c834 --- /dev/null +++ b/puppet/environment/vagrant/modules/sklearn/manifests/init.pp @@ -0,0 +1,20 @@ +### +### init.pp: install sklearn, with all necessary dependencies. +### + +class sklearn { + ## create '/vagrant/build/' directory + contain system::build_directory + + ## install sklearn dependencies + contain package::sklearn_dependencies + + ## download scikit-learn + contain package::sklearn + + ## build scikit-learn + contain sklearn::build_sklearn + + ## install scikit-learn + contain sklearn::install_sklearn +} \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/system/manifests/init.pp b/puppet/environment/vagrant/modules/system/manifests/init.pp new file mode 100644 index 000000000..592a7cb32 --- /dev/null +++ b/puppet/environment/vagrant/modules/system/manifests/init.pp @@ -0,0 +1,7 @@ +### +### init.pp: configure system with general requirements. +### + +class system { + contain system::set_timezone +} \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/vagrant/manifests/init.pp b/puppet/environment/vagrant/modules/vagrant/manifests/init.pp new file mode 100644 index 000000000..feffc1910 --- /dev/null +++ b/puppet/environment/vagrant/modules/vagrant/manifests/init.pp @@ -0,0 +1,12 @@ +### +### init.pp: ensure 'vagrant-mounted' event fires, when '/vagrant' shared +### directory, is mounted within the guest virtual machine. +### + +class vagrant { + ## configure service + contain vagrant::service + + ## start service + contain vagrant::start +} \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/webserver/manifests/init.pp b/puppet/environment/vagrant/modules/webserver/manifests/init.pp new file mode 100644 index 000000000..1e4221982 --- /dev/null +++ b/puppet/environment/vagrant/modules/webserver/manifests/init.pp @@ -0,0 +1,16 @@ +### +### init.pp: start webserver. +### + +class webserver { + ## ensure log directory + require system::log_directory + + ## install webserver + contain webserver::service + + ## start webservers + contain package::pyyaml + contain package::gunicorn + contain webserver::start +} \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/webserver/manifests/service.pp b/puppet/environment/vagrant/modules/webserver/manifests/service.pp index e08cac228..a5005b87a 100644 --- a/puppet/environment/vagrant/modules/webserver/manifests/service.pp +++ b/puppet/environment/vagrant/modules/webserver/manifests/service.pp @@ -2,6 +2,9 @@ ### service.pp, configure webserver(s), and corresponding proxy. ### class webserver::service { + include compiler::initial_compile + include webserver::start + ## variables $hiera_general = lookup('general') $hiera_development = lookup('development') @@ -24,10 +27,10 @@ $nginx_version = $hiera_development['apt']['nginx'] $nginx_proxy = "${nginx_reverse_proxy['proxy']}:${gunicorn_port}" - ## include webserver dependencies - include python - include python::flask - include python::requests + ## contain webserver dependencies + contain python + contain python::flask + contain python::requests ## nginx: installation class { 'nginx': @@ -45,5 +48,7 @@ file { '/etc/init/start_gunicorn.conf': ensure => file, content => dos2unix(template($template_path)), + require => Class['compiler::initial_compile'], + notify => Class['webserver::start'], } } \ No newline at end of file diff --git a/puppet/environment/vagrant/modules/webserver/manifests/start.pp b/puppet/environment/vagrant/modules/webserver/manifests/start.pp index 9d952a637..f5354a85c 100644 --- a/puppet/environment/vagrant/modules/webserver/manifests/start.pp +++ b/puppet/environment/vagrant/modules/webserver/manifests/start.pp @@ -2,6 +2,8 @@ ### start.pp, ensure gunicorn webserver workers running. ### class webserver::start { + include webserver::service + ## variables $hiera_general = lookup('general') $vagrant_mounted = $hiera_general['vagrant_implement'] @@ -10,14 +12,16 @@ if $vagrant_mounted { # ensure service starts at boot service { 'start_gunicorn': - ensure => 'running', - enable => true, + ensure => 'running', + enable => true, + require => Class['webserver::service'], } } else { ## run and restart when needed service { 'start_gunicorn': - ensure => 'running', + ensure => 'running', + require => Class['webserver::service'], } } } \ No newline at end of file diff --git a/redis.dockerfile b/redis.dockerfile index 3112b56e1..2f1c88788 100644 --- a/redis.dockerfile +++ b/redis.dockerfile @@ -6,7 +6,7 @@ ENV ENVIRONMENT docker ENV ENVIRONMENT_DIR $ROOT_PROJECT/puppet/environment/$ENVIRONMENT ## provision with puppet -RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/manifests/configure_redis.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test +RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/modules/redis/manifests/init.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test ## executed everytime container starts -CMD ["redis-server"] +CMD ["/bin/sh", "-c", "redis-server"] diff --git a/webserver.dockerfile b/webserver.dockerfile index 78e617b3a..938db43a4 100644 --- a/webserver.dockerfile +++ b/webserver.dockerfile @@ -6,7 +6,7 @@ ENV ENVIRONMENT docker ENV ENVIRONMENT_DIR $ROOT_PROJECT/puppet/environment/$ENVIRONMENT ## provision with puppet -RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/manifests/start_webserver.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test +RUN /opt/puppetlabs/bin/puppet apply $ENVIRONMENT_DIR/modules/webserver/manifests/init.pp --modulepath=$ENVIRONMENT_DIR/modules_contrib:$ENVIRONMENT_DIR/modules --confdir=$ROOT_PROJECT/hiera/test ## executed everytime container starts WORKDIR /var/machine-learning