summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/gitlab/git/repository.rb2
-rw-r--r--lib/gitlab/git/tree.rb23
-rw-r--r--lib/gitlab/git_access.rb2
-rw-r--r--lib/gitlab/gitaly_client.rb23
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb5
-rw-r--r--lib/gitlab/job_waiter.rb8
-rw-r--r--lib/gitlab/sidekiq_middleware/memory_killer.rb67
-rw-r--r--lib/gitlab/sidekiq_middleware/shutdown.rb133
8 files changed, 187 insertions, 76 deletions
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index e3cbf017e55..ddb9cf433eb 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -2206,7 +2206,7 @@ module Gitlab
with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do
# Apply diff of the `diff_range` to the worktree
diff = run_git!(%W(diff --binary #{diff_range}))
- run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin|
+ run_git!(%w(apply --index --whitespace=nowarn), chdir: squash_path, env: env) do |stdin|
stdin.binmode
stdin.write(diff)
end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index ba6058fd3c9..b6ceb542dd1 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -14,14 +14,14 @@ module Gitlab
# Uses rugged for raw objects
#
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320
- def where(repository, sha, path = nil)
+ def where(repository, sha, path = nil, recursive = false)
path = nil if path == '' || path == '/'
Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled|
if is_enabled
- repository.gitaly_commit_client.tree_entries(repository, sha, path)
+ repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive)
else
- tree_entries_from_rugged(repository, sha, path)
+ tree_entries_from_rugged(repository, sha, path, recursive)
end
end
end
@@ -57,7 +57,22 @@ module Gitlab
end
end
- def tree_entries_from_rugged(repository, sha, path)
+ def tree_entries_from_rugged(repository, sha, path, recursive)
+ current_path_entries = get_tree_entries_from_rugged(repository, sha, path)
+ ordered_entries = []
+
+ current_path_entries.each do |entry|
+ ordered_entries << entry
+
+ if recursive && entry.dir?
+ ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true))
+ end
+ end
+
+ ordered_entries
+ end
+
+ def get_tree_entries_from_rugged(repository, sha, path)
commit = repository.lookup(sha)
root_tree = commit.tree
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index bbdb593d4e2..6400089a22f 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -199,7 +199,7 @@ module Gitlab
def check_repository_existence!
unless repository.exists?
- raise UnauthorizedError, ERROR_MESSAGES[:no_repo]
+ raise NotFoundError, ERROR_MESSAGES[:no_repo]
end
end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index c5d3e944f7d..9cd76630484 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -125,6 +125,8 @@ module Gitlab
kwargs = yield(kwargs) if block_given?
stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
+ rescue GRPC::Unavailable => ex
+ handle_grpc_unavailable!(ex)
ensure
duration = Gitlab::Metrics::System.monotonic_time - start
@@ -135,6 +137,27 @@ module Gitlab
duration)
end
+ def self.handle_grpc_unavailable!(ex)
+ status = ex.to_status
+ raise ex unless status.details == 'Endpoint read failed'
+
+ # There is a bug in grpc 1.8.x that causes a client process to get stuck
+ # always raising '14:Endpoint read failed'. The only thing that we can
+ # do to recover is to restart the process.
+ #
+ # See https://gitlab.com/gitlab-org/gitaly/issues/1029
+
+ if Sidekiq.server?
+ raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s)
+ else
+ # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request.
+ Process.kill('QUIT', Process.pid)
+ end
+
+ raise ex
+ end
+ private_class_method :handle_grpc_unavailable!
+
def self.current_transaction_labels
Gitlab::Metrics::Transaction.current&.labels || {}
end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 269a048cf5d..d60f57717b5 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -105,11 +105,12 @@ module Gitlab
entry unless entry.oid.blank?
end
- def tree_entries(repository, revision, path)
+ def tree_entries(repository, revision, path, recursive)
request = Gitaly::GetTreeEntriesRequest.new(
repository: @gitaly_repo,
revision: encode_binary(revision),
- path: path.present? ? encode_binary(path) : '.'
+ path: path.present? ? encode_binary(path) : '.',
+ recursive: recursive
)
response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout)
diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb
index f654508c391..f7a8eae0be4 100644
--- a/lib/gitlab/job_waiter.rb
+++ b/lib/gitlab/job_waiter.rb
@@ -15,16 +15,22 @@ module Gitlab
# push to that array when done. Once the waiter has popped `count` items, it
# knows all the jobs are done.
class JobWaiter
+ KEY_PREFIX = "gitlab:job_waiter".freeze
+
def self.notify(key, jid)
Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) }
end
+ def self.key?(key)
+ key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/
+ end
+
attr_reader :key, :finished
attr_accessor :jobs_remaining
# jobs_remaining - the number of jobs left to wait for
# key - The key of this waiter.
- def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}")
+ def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}")
@key = key
@jobs_remaining = jobs_remaining
@finished = []
diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb
deleted file mode 100644
index b89ae2505c9..00000000000
--- a/lib/gitlab/sidekiq_middleware/memory_killer.rb
+++ /dev/null
@@ -1,67 +0,0 @@
-module Gitlab
- module SidekiqMiddleware
- class MemoryKiller
- # Default the RSS limit to 0, meaning the MemoryKiller is disabled
- MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
- # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
- GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
- # Wait 30 seconds for running jobs to finish during graceful shutdown
- SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
-
- # Create a mutex used to ensure there will be only one thread waiting to
- # shut Sidekiq down
- MUTEX = Mutex.new
-
- def call(worker, job, queue)
- yield
-
- current_rss = get_rss
-
- return unless MAX_RSS > 0 && current_rss > MAX_RSS
-
- Thread.new do
- # Return if another thread is already waiting to shut Sidekiq down
- return unless MUTEX.try_lock
-
- Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\
- " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}"
- Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later"
-
- # Wait `GRACE_TIME` to give the memory intensive job time to finish.
- # Then, tell Sidekiq to stop fetching new jobs.
- wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs')
-
- # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish.
- # Then, tell Sidekiq to gracefully shut down by giving jobs a few more
- # moments to finish, killing and requeuing them if they didn't, and
- # then terminating itself.
- wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down')
-
- # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't.
- wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die')
- end
- end
-
- private
-
- def get_rss
- output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s)
- return 0 unless status.zero?
-
- output.to_i
- end
-
- def wait_and_signal(time, signal, explanation)
- Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
- sleep(time)
-
- Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
- Process.kill(signal, pid)
- end
-
- def pid
- Process.pid
- end
- end
- end
-end
diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb
new file mode 100644
index 00000000000..c2b8d6de66e
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/shutdown.rb
@@ -0,0 +1,133 @@
+require 'mutex_m'
+
+module Gitlab
+ module SidekiqMiddleware
+ class Shutdown
+ extend Mutex_m
+
+ # Default the RSS limit to 0, meaning the MemoryKiller is disabled
+ MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i
+ # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
+ GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i
+ # Wait 30 seconds for running jobs to finish during graceful shutdown
+ SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i
+
+ # This exception can be used to request that the middleware start shutting down Sidekiq
+ WantShutdown = Class.new(StandardError)
+
+ ShutdownWithoutRaise = Class.new(WantShutdown)
+ private_constant :ShutdownWithoutRaise
+
+ # For testing only, to avoid race conditions (?) in Rspec mocks.
+ attr_reader :trace
+
+ # We store the shutdown thread in a class variable to ensure that there
+ # can be only one shutdown thread in the process.
+ def self.create_shutdown_thread
+ mu_synchronize do
+ return unless @shutdown_thread.nil?
+
+ @shutdown_thread = Thread.new { yield }
+ end
+ end
+
+ # For testing only: so we can wait for the shutdown thread to finish.
+ def self.shutdown_thread
+ mu_synchronize { @shutdown_thread }
+ end
+
+ # For testing only: so that we can reset the global state before each test.
+ def self.clear_shutdown_thread
+ mu_synchronize { @shutdown_thread = nil }
+ end
+
+ def initialize
+ @trace = Queue.new if Rails.env.test?
+ end
+
+ def call(worker, job, queue)
+ shutdown_exception = nil
+
+ begin
+ yield
+ check_rss!
+ rescue WantShutdown => ex
+ shutdown_exception = ex
+ end
+
+ return unless shutdown_exception
+
+ self.class.create_shutdown_thread do
+ do_shutdown(worker, job, shutdown_exception)
+ end
+
+ raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise)
+ end
+
+ private
+
+ def do_shutdown(worker, job, shutdown_exception)
+ Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\
+ "#{worker.class} JID-#{job['jid']}"
+ Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later"
+
+ # Wait `GRACE_TIME` to give the memory intensive job time to finish.
+ # Then, tell Sidekiq to stop fetching new jobs.
+ wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs')
+
+ # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish.
+ # Then, tell Sidekiq to gracefully shut down by giving jobs a few more
+ # moments to finish, killing and requeuing them if they didn't, and
+ # then terminating itself.
+ wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down')
+
+ # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't.
+ wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die')
+ end
+
+ def check_rss!
+ return unless MAX_RSS > 0
+
+ current_rss = get_rss
+ return unless current_rss > MAX_RSS
+
+ raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}")
+ end
+
+ def get_rss
+ output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s)
+ return 0 unless status.zero?
+
+ output.to_i
+ end
+
+ def wait_and_signal(time, signal, explanation)
+ Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
+ sleep(time)
+
+ Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})"
+ kill(signal, pid)
+ end
+
+ def pid
+ Process.pid
+ end
+
+ def sleep(time)
+ if Rails.env.test?
+ @trace << [:sleep, time]
+ else
+ Kernel.sleep(time)
+ end
+ end
+
+ def kill(signal, pid)
+ if Rails.env.test?
+ @trace << [:kill, signal, pid]
+ else
+ Process.kill(signal, pid)
+ end
+ end
+ end
+ end
+end