Merge branch 'sidekiq-memory-killer-shutdown-signal' into 'master'

Add SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL env var

It looks like SIGTERM may not be enough to shut down a Sidekiq process
when its RSS has gotten too big. This change will allow us to experiment
with sending SIGKILL instead of SIGTERM to Sidekiq processes on
gitlab.com.

See merge request !1812

1 files changed, 6 insertions, 5 deletions

diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb
index 0f2db50e98c..f33b2dedf4a 100644
--- a/lib/gitlab/sidekiq_middleware/memory_killer.rb
+++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb
@@ -7,6 +7,7 @@ module Gitlab
       GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
       # Wait 30 seconds for running jobs to finish during graceful shutdown
       SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
+      SHUTDOWN_SIGNAL = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL'] || 'SIGTERM').to_s
 
       # Create a mutex used to ensure there will be only one thread waiting to
       # shut Sidekiq down
@@ -24,19 +25,19 @@ module Gitlab
 
           Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
             "#{MAX_RSS}"
-          Sidekiq.logger.warn "spawned thread that will shut down PID "\
-            "#{Process.pid} in #{GRACE_TIME} seconds"
+          Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\
+            "in #{GRACE_TIME} seconds"
           sleep(GRACE_TIME)
 
           Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}"
           Process.kill('SIGUSR1', Process.pid)
 
           Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\
-            "SIGTERM to PID #{Process.pid}"
+            "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
           sleep(SHUTDOWN_WAIT)
 
-          Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}"
-          Process.kill('SIGTERM', Process.pid)
+          Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
+          Process.kill(SHUTDOWN_SIGNAL, Process.pid)
         end
       end