Skip to content
This repository has been archived by the owner on Dec 31, 2022. It is now read-only.

Commit

Permalink
feat(worker): Safely quiet and shutdown sidekiq
Browse files Browse the repository at this point in the history
  • Loading branch information
Kevin Pheasey committed Mar 3, 2017
1 parent 0f4f64c commit c36652f
Show file tree
Hide file tree
Showing 5 changed files with 103 additions and 22 deletions.
8 changes: 8 additions & 0 deletions libraries/drivers_worker_base.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,14 @@ def restart_monit
end
end

def unmonitor_monit
(1..process_count).each do |process_number|
context.execute "monit unmonitor #{adapter}_#{app['shortname']}-#{process_number}" do
retries 3
end
end
end

def process_count
[out[:process_count].to_i, 1].max
end
Expand Down
44 changes: 44 additions & 0 deletions libraries/drivers_worker_sidekiq.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,20 @@ def configure
add_worker_monit
end

def before_deploy
quiet_sidekiq
end

def after_deploy
restart_monit
end

def shutdown
quiet_sidekiq
unmonitor_monit
stop_sidekiq
end

alias after_undeploy after_deploy

private
Expand All @@ -33,6 +44,39 @@ def add_sidekiq_config
end
end

def quiet_sidekiq
(1..process_count).each do |process_number|
pid_file = pid_file(process_number)
Chef::Log.info("Quiet Sidekiq process if exists: #{pid_file}")
next unless File.file?(pid_file) && pid_exists?(File.open(pid_file).read)
context.execute "/bin/su - #{node['deployer']['user']} -c 'kill -s USR1 `cat #{pid_file}`'"
end
end

def stop_sidekiq
(1..process_count).each do |process_number|
pid_file = pid_file(process_number)
timeout = (out[:config]['timeout'] || 8).to_i

context.execute(
"/bin/su - #{node['deployer']['user']} -c 'cd #{File.join(deploy_dir(app), 'current')} && " \
"#{environment.map { |k, v| "#{k}=\"#{v}\"" }.join(' ')} " \
"bundle exec sidekiqctl stop #{pid_file} #{timeout}'"
)
end
end

def pid_file(process_number)
"#{deploy_dir(app)}/shared/pids/sidekiq_#{app['shortname']}-#{process_number}.pid"
end

def pid_exists?(pid)
Process.getpgid(pid.to_i)
true
rescue Errno::ESRCH
false
end

def configuration
JSON.parse(out[:config].to_json, symbolize_names: true)
end
Expand Down
28 changes: 16 additions & 12 deletions spec/unit/recipes/configure_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,16 +210,17 @@
'start program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiq ' \
'-C /srv/www/dummy_project/shared/config/sidekiq_1.yml ' \
'-i 0 ' \
'-P /srv/www/dummy_project/shared/pids/sidekiq_dummy_project-1.pid ' \
'-r /srv/www/dummy_project/current/lorem_ipsum.rb 2>&1 ' \
'| logger -t sidekiq-dummy_project-1\'" with timeout 90 seconds'
)
expect(chef_run_rhel)
.to render_file("/etc/monit.d/sidekiq_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'stop program = "/bin/su - deploy -c ' \
'\'kill -s TERM `cat /srv/www/dummy_project/shared/pids/sidekiq_dummy_project-1.pid`\'' \
'" with timeout 90 seconds'
'stop program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiqctl stop ' \
'/srv/www/dummy_project/shared/pids/sidekiq_dummy_project-1.pid 8\'" with timeout 18 seconds'
)
expect(chef_run_rhel)
.to render_file("/etc/monit.d/sidekiq_#{aws_opsworks_app['shortname']}.monitrc")
Expand All @@ -233,16 +234,17 @@
'start program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiq ' \
'-C /srv/www/dummy_project/shared/config/sidekiq_2.yml ' \
'-i 1 ' \
'-P /srv/www/dummy_project/shared/pids/sidekiq_dummy_project-2.pid ' \
'-r /srv/www/dummy_project/current/lorem_ipsum.rb 2>&1 ' \
'| logger -t sidekiq-dummy_project-2\'" with timeout 90 seconds'
)
expect(chef_run_rhel)
.to render_file("/etc/monit.d/sidekiq_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'stop program = "/bin/su - deploy -c ' \
'\'kill -s TERM `cat /srv/www/dummy_project/shared/pids/sidekiq_dummy_project-2.pid`\'' \
'" with timeout 90 seconds'
'stop program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiqctl stop ' \
'/srv/www/dummy_project/shared/pids/sidekiq_dummy_project-2.pid 8\'" with timeout 18 seconds'
)
expect(chef_run_rhel)
.to render_file("/etc/monit.d/sidekiq_#{aws_opsworks_app['shortname']}.monitrc")
Expand All @@ -266,16 +268,17 @@
'start program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiq ' \
'-C /srv/www/dummy_project/shared/config/sidekiq_1.yml ' \
'-i 0 ' \
'-P /srv/www/dummy_project/shared/pids/sidekiq_dummy_project-1.pid ' \
'-r /srv/www/dummy_project/current/lorem_ipsum.rb 2>&1 ' \
'| logger -t sidekiq-dummy_project-1\'" with timeout 90 seconds'
)
expect(chef_run)
.to render_file("/etc/monit/conf.d/sidekiq_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'stop program = "/bin/su - deploy -c ' \
'\'kill -s TERM `cat /srv/www/dummy_project/shared/pids/sidekiq_dummy_project-1.pid`\'' \
'" with timeout 90 seconds'
'stop program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiqctl stop ' \
'/srv/www/dummy_project/shared/pids/sidekiq_dummy_project-1.pid 8\'" with timeout 18 seconds'
)
expect(chef_run)
.to render_file("/etc/monit/conf.d/sidekiq_#{aws_opsworks_app['shortname']}.monitrc")
Expand All @@ -289,16 +292,17 @@
'start program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiq ' \
'-C /srv/www/dummy_project/shared/config/sidekiq_2.yml ' \
'-i 1 ' \
'-P /srv/www/dummy_project/shared/pids/sidekiq_dummy_project-2.pid ' \
'-r /srv/www/dummy_project/current/lorem_ipsum.rb 2>&1 ' \
'| logger -t sidekiq-dummy_project-2\'" with timeout 90 seconds'
)
expect(chef_run)
.to render_file("/etc/monit/conf.d/sidekiq_#{aws_opsworks_app['shortname']}.monitrc")
.with_content(
'stop program = "/bin/su - deploy -c ' \
'\'kill -s TERM `cat /srv/www/dummy_project/shared/pids/sidekiq_dummy_project-2.pid`\'' \
'" with timeout 90 seconds'
'stop program = "/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiqctl stop ' \
'/srv/www/dummy_project/shared/pids/sidekiq_dummy_project-2.pid 8\'" with timeout 18 seconds'
)
expect(chef_run)
.to render_file("/etc/monit/conf.d/sidekiq_#{aws_opsworks_app['shortname']}.monitrc")
Expand Down
24 changes: 24 additions & 0 deletions spec/unit/recipes/shutdown_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,30 @@
end.not_to raise_error
end

context 'safely shutdown sidekiq' do
it 'unmonitors sidekiq processes' do
expect(chef_run).to run_execute('monit unmonitor sidekiq_dummy_project-1')
expect(chef_run).to run_execute('monit unmonitor sidekiq_dummy_project-2')
end

it 'shutsdown sidekiq processes' do
expect(chef_run).to(
run_execute(
'/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiqctl stop ' \
'/srv/www/dummy_project/shared/pids/sidekiq_dummy_project-1.pid 8\''
)
)
expect(chef_run).to(
run_execute(
'/bin/su - deploy -c \'cd /srv/www/dummy_project/current && ENV_VAR1="test" ' \
'ENV_VAR2="some data" RAILS_ENV="staging" bundle exec sidekiqctl stop ' \
'/srv/www/dummy_project/shared/pids/sidekiq_dummy_project-2.pid 8\''
)
)
end
end

it 'empty node[\'deploy\']' do
chef_run = ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '14.04') do |solo_node|
solo_node.set['lsb'] = node['lsb']
Expand Down
21 changes: 11 additions & 10 deletions templates/default/sidekiq.monitrc.erb
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
<% ([@out[:process_count].to_i || 1].max).times do |n| %>
<% identifier = "#{@application}-#{n+1}" %>
<% conf_file = "#{@deploy_to}/shared/config/sidekiq_#{n+1}.yml" %>
<% pid_file = "#{@deploy_to}/shared/pids/sidekiq_#{identifier}.pid" %>
<% syslog = !!@out[:syslog] ? "2>&1 | logger -t sidekiq-#{identifier}" : '' %>
<% to_require = @out[:require].present? ? " -r #{File.join(@deploy_to, 'current', @out[:require])}" : '' %>
<% identifier = "#{@application}-#{n+1}" %>
<% conf_file = "#{@deploy_to}/shared/config/sidekiq_#{n+1}.yml" %>
<% pid_file = "#{@deploy_to}/shared/pids/sidekiq_#{identifier}.pid" %>
<% syslog = !!@out[:syslog] ? "2>&1 | logger -t sidekiq-#{identifier}" : '' %>
<% to_require = @out[:require].present? ? " -r #{File.join(@deploy_to, 'current', @out[:require])}" : '' %>
<% timeout = (@out[:config]['timeout'] || 8).to_i %>

check process sidekiq_<%= identifier.to_s %>
with pidfile <%= pid_file.to_s %>
start program = "/bin/su - <%= node['deployer']['user'] %> -c 'cd <%= File.join(@deploy_to, 'current') %> && <%= @environment.map {|k,v| "#{k}=\"#{v}\""}.join(' ') %> bundle exec sidekiq -C <%= conf_file.to_s %> -P <%= pid_file.to_s %><%= to_require.to_s %> <%= syslog.to_s %>'" with timeout 90 seconds
stop program = "/bin/su - <%= node['deployer']['user'] %> -c 'kill -s TERM `cat <%= pid_file.to_s %>`'" with timeout 90 seconds
group sidekiq_<%= @application.to_s %>_group
check process sidekiq_<%= identifier.to_s %>
with pidfile <%= pid_file.to_s %>
start program = "/bin/su - <%= node['deployer']['user'] %> -c 'cd <%= File.join(@deploy_to, 'current') %> && <%= @environment.map { |k, v| "#{k}=\"#{v}\"" }.join(' ') %> bundle exec sidekiq -C <%= conf_file.to_s %> -i <%= n %> -P <%= pid_file.to_s %><%= to_require.to_s %> <%= syslog.to_s %>'" with timeout 90 seconds
stop program = "/bin/su - <%= node['deployer']['user'] %> -c 'cd <%= File.join(@deploy_to, 'current') %> && <%= @environment.map { |k, v| "#{k}=\"#{v}\"" }.join(' ') %> bundle exec sidekiqctl stop <%= pid_file %> <%= timeout %>'" with timeout <%= timeout + 10 %> seconds
group sidekiq_<%= @application.to_s %>_group

<% end %>

0 comments on commit c36652f

Please sign in to comment.