From 1c1f18b416febb8296facca37dc7504c2da880f8 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 7 May 2015 18:47:03 +0200 Subject: Add SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL env var It looks like SIGTERM may not be enough to shut down a Sidekiq process when its RSS has gotten too big. This change will allow us to experiment with sending SIGKILL instead of SIGTERM to Sidekiq processes on gitlab.com. --- lib/gitlab/sidekiq_middleware/memory_killer.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 0f2db50e98c..f33b2dedf4a 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -7,6 +7,7 @@ module Gitlab GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i # Wait 30 seconds for running jobs to finish during graceful shutdown SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + SHUTDOWN_SIGNAL = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL'] || 'SIGTERM').to_s # Create a mutex used to ensure there will be only one thread waiting to # shut Sidekiq down @@ -24,19 +25,19 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "spawned thread that will shut down PID "\ - "#{Process.pid} in #{GRACE_TIME} seconds" + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\ + "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" Process.kill('SIGUSR1', Process.pid) Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ - "SIGTERM to PID #{Process.pid}" + "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}" sleep(SHUTDOWN_WAIT) - Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" - Process.kill('SIGTERM', Process.pid) + Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + Process.kill(SHUTDOWN_SIGNAL, Process.pid) end end -- cgit v1.2.1