Fixes #28067 - dynflow sidekiq services config

.fixtures.yml

@@ -13,6 +13,7 @@ fixtures:
     mysql:            'https://github.com/puppetlabs/puppetlabs-mysql'
     postgresql:       'https://github.com/puppetlabs/puppetlabs-postgresql'
     puppet:           'https://github.com/theforeman/puppet-puppet'
+    redis:            'https://github.com/voxpupuli/puppet-redis'
     systemd:          'https://github.com/camptocamp/puppet-systemd'
     selinux_core:
       repo: "https://github.com/puppetlabs/puppetlabs-selinux_core"

manifests/config.pp

@@ -80,6 +80,34 @@
     ensure  => absent,
   }
 
+  if $::foreman::jobs_sidekiq {
+    if $::foreman::jobs_sidekiq_redis_url == undef {
+      include ::redis'
+
+      redis::instance { 'dynflow':
+        port => $::foreman::jobs_sidekiq_redis_port,
+      }
+    }
+
+    file { '/etc/foreman/dynflow/':
+      ensure  => directory,
+    }
+
+    file { '/etc/foreman/dynflow/orchestrator.yml':
+      owner   => 'root',
+      group   => $::foreman::group,
+      mode    => '0640',
+      content => template('foreman/dynflow_orchestrator.yml.erb'),
+    }
+
+    file { '/etc/foreman/dynflow/worker.yml':
+      owner   => 'root',
+      group   => $::foreman::group,
+      mode    => '0640',
+      content => template('foreman/dynflow_worker.yml.erb'),
+    }
+  }
+
   if $::foreman::apache  {
     contain foreman::config::apache
 

manifests/init.pp

@@ -204,6 +204,12 @@
 #
 # $jobs_service_ensure::          Whether the Dynflow executor should be running or stopped
 #
+# $jobs_sidekiq::                 Whether use Sidekiq to run Dynflow executor
+#
+# $jobs_sidekiq_redis_url::       If set, the redis server is not managed and we use the defined url to connect
+#
+# $jobs_sidekiq_redis_port::      What port should redis server for Dynflow worker use
+#
 # $hsts_enabled::                 Should HSTS enforcement in https requests be enabled
 #
 # $cors_domains::                 List of domains that show be allowed for Cross-Origin Resource Sharing. This requires Foreman 1.22+
@@ -305,6 +311,9 @@
   String $jobs_service = $::foreman::params::jobs_service,
   Stdlib::Ensure::Service $jobs_service_ensure = $::foreman::params::jobs_service_ensure,
   Boolean $jobs_service_enable = $::foreman::params::jobs_service_enable,
+  Boolean $jobs_sidekiq = $::foreman::params::jobs_sidekiq,
+  Optional[Redis::RedisUrl] $jobs_sidekiq_redis_url = $::foreman::params::jobs_sidekiq_redis_url,
+  Stdlib::Port $jobs_sidekiq_redis_port = $::foreman::params::jobs_sidekiq_redis_port,
   Boolean $hsts_enabled = $::foreman::params::hsts_enabled,
   Array[Stdlib::HTTPUrl] $cors_domains = $::foreman::params::cors_domains,
 ) inherits foreman::params {

manifests/install.pp

@@ -37,6 +37,12 @@
     }
   }
 
+  if $::foreman::jobs_sidekiq {
+    package { 'foreman-dynflow-sidekiq':
+      ensure => installed,
+    }
+  }
+
   if $::foreman::use_foreman_service {
     package { 'foreman-service':
       ensure => installed,

manifests/params.pp

@@ -95,6 +95,9 @@
   $jobs_service = 'dynflowd'
   $jobs_service_ensure = 'running'
   $jobs_service_enable = true
+  $jobs_sidekiq = true
+  $jobs_sidekiq_redis_url = undef
+  $jobs_sidekiq_redis_port = 6381
 
   $hsts_enabled = true
 

manifests/service.pp

@@ -11,10 +11,30 @@
   String $jobs_service = $::foreman::jobs_service,
   Stdlib::Ensure::Service $jobs_service_ensure = $::foreman::jobs_service_ensure,
   Boolean $jobs_service_enable = $::foreman::jobs_service_enable,
+  Boolean $jobs_sidekiq = $::foreman::jobs_sidekiq,
 ) {
+  if $jobs_sidekiq {
+    $sidekiq_ensure = $jobs_service_ensure
+    $sidekiq_enable = $jobs_service_enable
+    $dynflowd_ensure = 'stopped'
+    $dynflowd_enable = false
+  } else {
+    $dynflowd_ensure = $jobs_service_ensure
+    $dynflowd_enable = $jobs_service_enable
+    $sidekiq_enable = false
+    $sidekiq_ensure = 'stopped'
+  }
+  service { 'dynflow-sidekiq@orchestrator':
+    ensure => $sidekiq_ensure,
+    enable => $sidekiq_enable,
+  }
+  service { 'dynflow-sidekiq@worker':
+    ensure => $sidekiq_ensure,
+    enable => $sidekiq_enable,
+  }
   service { $jobs_service:
-    ensure => $jobs_service_ensure,
-    enable => $jobs_service_enable,
+    ensure => $dynflowd_ensure,
+    enable => $dynflowd_enable,
   }
 
   if $apache {

metadata.json

@@ -35,6 +35,10 @@
     {
       "name": "puppet/extlib",
       "version_requirement": ">= 3.0.0 < 5.0.0"
+    },
+    {
+      "name": "puppet/redis",
+      "version_requirement": ">= 4.2.0 < 5.0.0"
     }
   ],
   "requirements": [

spec/classes/foreman_install_spec.rb

@@ -19,6 +19,17 @@
         it { should contain_package('foreman-postgresql').that_requires('Class[foreman::repo]') }
       end
 
+      describe 'sidekiq jobs' do
+        context 'with jobs_sidekiq disabled' do
+          let(:params) { super().merge(jobs_sidekiq: false) }
+          it { is_expected.not_to contain_package('foreman-dynflow-sidekiq') }
+        end
+        context 'with jobs_sidekiq enabled' do
+          let(:params) { super().merge(jobs_sidekiq: true) }
+          it { is_expected.to contain_package('foreman-dynflow-sidekiq') }
+        end
+      end
+
       context 'with SELinux enabled' do
         let(:facts) { super().merge(selinux: true) }
 

spec/classes/foreman_service_spec.rb

@@ -17,7 +17,8 @@
       foreman_service_enable: true,
       jobs_service: 'dynflower',
       jobs_service_ensure: 'stopped',
-      jobs_service_enable: false
+      jobs_service_enable: false,
+      jobs_sidekiq: false
     }
   end
 
@@ -67,4 +68,12 @@
         .with_enable(true)
     end
   end
+
+  context 'with sidekiq' do
+    let(:pre_condition) { 'include ::apache' }
+    let(:params) { super().merge(jobs_sidekiq: true, jobs_service_ensure: 'running', jobs_service_enable: true) }
+    it { is_expected.to contain_service('dynflow-sidekiq@orchestrator').with_ensure('running').with_enable(true) }
+    it { is_expected.to contain_service('dynflow-sidekiq@worker').with_ensure('running').with_enable(true) }
+    it { is_expected.to contain_service('dynflower').with_ensure('stopped').with_enable(false) }
+  end
 end

spec/classes/foreman_spec.rb

@@ -134,7 +134,9 @@
         # service
         it { should contain_class('foreman::service') }
         it { should_not contain_service('foreman') }
-        it { is_expected.to contain_service('dynflowd').with_ensure('running').with_enable(true) }
+        it { is_expected.to contain_service('dynflow-sidekiq@orchestrator').with_ensure('running').with_enable(true) }
+        it { is_expected.to contain_service('dynflow-sidekiq@worker').with_ensure('running').with_enable(true) }
+        it { is_expected.to contain_service('dynflowd').with_ensure('stopped').with_enable(false) }
 
         it 'should restart passenger' do
           should contain_exec('restart_foreman')

templates/dynflow_orchestrator.yml.erb

@@ -0,0 +1,3 @@
+:concurrency: 1
+:queues:
+  - dynflow_orchestrator

templates/dynflow_worker.yml.erb

@@ -0,0 +1,4 @@
+:concurrency: <%= scope.lookupvar("foreman::dynflow_pool_size") %>
+:queues:
+  - default
+  - remote_execution

templates/settings.yaml.erb

@@ -74,6 +74,9 @@
 
 :dynflow:
   :pool_size: <%= scope.lookupvar("foreman::dynflow_pool_size") %>
+<% if scope.lookupvar("foreman::jobs_sidekiq") %>
+  :redis_url: <%= scope.lookupvar("foreman::jobs_sidekiq_redis_url") || "redis://localhost:#{scope.lookupvar("foreman::jobs_sidekiq_redis_port")}/0" %>
+<% end %>
 <% if scope.lookupvar("foreman::apache") && !scope.lookupvar("foreman::passenger") -%>
 
 # Configure reverse proxy headers