diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/backup/repository.rb | 2 | ||||
| -rw-r--r-- | lib/ci/api/api.rb | 39 | ||||
| -rw-r--r-- | lib/ci/api/builds.rb | 219 | ||||
| -rw-r--r-- | lib/ci/api/entities.rb | 93 | ||||
| -rw-r--r-- | lib/ci/api/helpers.rb | 89 | ||||
| -rw-r--r-- | lib/ci/api/runners.rb | 50 | ||||
| -rw-r--r-- | lib/ci/api/triggers.rb | 39 | ||||
| -rw-r--r-- | lib/gitlab/background_migration/migrate_stage_status.rb | 77 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 35 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/commit_service.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/import_export/import_export.yml | 1 | ||||
| -rw-r--r-- | lib/gitlab/job_waiter.rb | 57 | ||||
| -rw-r--r-- | lib/gitlab/sidekiq_throttler.rb | 2 | ||||
| -rw-r--r-- | lib/tasks/gitlab/import.rake | 6 |
14 files changed, 149 insertions, 564 deletions
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 88821ae56e0..4e92be85110 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -75,7 +75,7 @@ module Backup path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - project.ensure_storage_path_exist + project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb deleted file mode 100644 index 24bb3649a76..00000000000 --- a/lib/ci/api/api.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Ci - module API - class API < Grape::API - include ::API::APIGuard - version 'v1', using: :path - - rescue_from ActiveRecord::RecordNotFound do - rack_response({ 'message' => '404 Not found' }.to_json, 404) - end - - # Retain 405 error rather than a 500 error for Grape 0.15.0+. - # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes - rescue_from Grape::Exceptions::MethodNotAllowed do |e| - error! e.message, e.status, e.headers - end - - rescue_from Grape::Exceptions::Base do |e| - error! e.message, e.status, e.headers - end - - rescue_from :all do |exception| - handle_api_exception(exception) - end - - content_type :txt, 'text/plain' - content_type :json, 'application/json' - format :json - - helpers ::SentryHelper - helpers ::Ci::API::Helpers - helpers ::API::Helpers - helpers Gitlab::CurrentSettings - - mount ::Ci::API::Builds - mount ::Ci::API::Runners - mount ::Ci::API::Triggers - end - end -end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb deleted file mode 100644 index 79058c02ce5..00000000000 --- a/lib/ci/api/builds.rb +++ /dev/null @@ -1,219 +0,0 @@ -module Ci - module API - # Builds API - class Builds < Grape::API - resource :builds do - # Runs oldest pending build by runner - Runners only - # - # Parameters: - # token (required) - The uniq token of runner - # - # Example Request: - # POST /builds/register - post "register" do - authenticate_runner! - required_attributes! [:token] - not_found! unless current_runner.active? - update_runner_info - - if current_runner.is_runner_queue_value_latest?(params[:last_update]) - header 'X-GitLab-Last-Update', params[:last_update] - Gitlab::Metrics.add_event(:build_not_found_cached) - return build_not_found! - end - - new_update = current_runner.ensure_runner_queue_value - - result = Ci::RegisterJobService.new(current_runner).execute - - if result.valid? - if result.build - Gitlab::Metrics.add_event(:build_found, - project: result.build.project.full_path) - - present result.build, with: Entities::BuildDetails - else - Gitlab::Metrics.add_event(:build_not_found) - - header 'X-GitLab-Last-Update', new_update - - build_not_found! - end - else - # We received build that is invalid due to concurrency conflict - Gitlab::Metrics.add_event(:build_invalid) - conflict! - end - end - - # Update an existing build - Runners only - # - # Parameters: - # id (required) - The ID of a project - # state (optional) - The state of a build - # trace (optional) - The trace of a build - # Example Request: - # PUT /builds/:id - put ":id" do - authenticate_runner! - build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - validate_build!(build) - - update_runner_info - - build.trace.set(params[:trace]) if params[:trace] - - Gitlab::Metrics.add_event(:update_build, - project: build.project.full_path) - - case params[:state].to_s - when 'success' - build.success - when 'failed' - build.drop - end - end - - # Send incremental log update - Runners only - # - # Parameters: - # id (required) - The ID of a build - # Body: - # content of logs to append - # Headers: - # Content-Range (required) - range of content that was sent - # BUILD-TOKEN (required) - The build authorization token - # Example Request: - # PATCH /builds/:id/trace.txt - patch ":id/trace.txt" do - build = authenticate_build! - - error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') - content_range = request.headers['Content-Range'] - content_range = content_range.split('-') - - stream_size = build.trace.append(request.body.read, content_range[0].to_i) - if stream_size < 0 - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) - end - - status 202 - header 'Build-Status', build.status - header 'Range', "0-#{stream_size}" - end - - # Authorize artifacts uploading for build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # filesize (optional) - the size of uploaded file - # Example Request: - # POST /builds/:id/artifacts/authorize - post ":id/artifacts/authorize" do - require_gitlab_workhorse! - Gitlab::Workhorse.verify_api_request!(headers) - not_allowed! unless Gitlab.config.artifacts.enabled - build = authenticate_build! - forbidden!('build is not running') unless build.running? - - if params[:filesize] - file_size = params[:filesize].to_i - file_to_large! unless file_size < max_artifacts_size - end - - status 200 - content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - Gitlab::Workhorse.artifact_upload_ok - end - - # Upload artifacts to build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # file (required) - Artifacts file - # expire_in (optional) - Specify when artifacts should expire (ex. 7d) - # Parameters (accelerated by GitLab Workhorse): - # file.path - path to locally stored body (generated by Workhorse) - # file.name - real filename as send in Content-Disposition - # file.type - real content type as send in Content-Type - # metadata.path - path to locally stored body (generated by Workhorse) - # metadata.name - filename (generated by Workhorse) - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Body: - # The file content - # - # Example Request: - # POST /builds/:id/artifacts - post ":id/artifacts" do - require_gitlab_workhorse! - not_allowed! unless Gitlab.config.artifacts.enabled - build = authenticate_build! - forbidden!('Build is not running!') unless build.running? - - artifacts_upload_path = ArtifactUploader.artifacts_upload_path - artifacts = uploaded_file(:file, artifacts_upload_path) - metadata = uploaded_file(:metadata, artifacts_upload_path) - - bad_request!('Missing artifacts file!') unless artifacts - file_to_large! unless artifacts.size < max_artifacts_size - - build.artifacts_file = artifacts - build.artifacts_metadata = metadata - build.artifacts_expire_in = - params['expire_in'] || - Gitlab::CurrentSettings.current_application_settings - .default_artifacts_expire_in - - if build.save - present(build, with: Entities::BuildDetails) - else - render_validation_error!(build) - end - end - - # Download the artifacts file from build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Example Request: - # GET /builds/:id/artifacts - get ":id/artifacts" do - build = authenticate_build! - artifacts_file = build.artifacts_file - - unless artifacts_file.exists? - not_found! - end - - unless artifacts_file.file_storage? - return redirect_to build.artifacts_file.url - end - - present_file!(artifacts_file.path, artifacts_file.filename) - end - - # Remove the artifacts file from build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Example Request: - # DELETE /builds/:id/artifacts - delete ":id/artifacts" do - build = authenticate_build! - - status(200) - build.erase_artifacts! - end - end - end - end -end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb deleted file mode 100644 index 31f66dd5a58..00000000000 --- a/lib/ci/api/entities.rb +++ /dev/null @@ -1,93 +0,0 @@ -module Ci - module API - module Entities - class Commit < Grape::Entity - expose :id, :sha, :project_id, :created_at - expose :status, :finished_at, :duration - expose :git_commit_message, :git_author_name, :git_author_email - end - - class CommitWithBuilds < Commit - expose :builds - end - - class ArtifactFile < Grape::Entity - expose :filename, :size - end - - class BuildOptions < Grape::Entity - expose :image - expose :services - expose :artifacts - expose :cache - expose :dependencies - expose :after_script - end - - class Build < Grape::Entity - expose :id, :ref, :tag, :sha, :status - expose :name, :token, :stage - expose :project_id - expose :project_name - expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } - end - - class BuildCredentials < Grape::Entity - expose :type, :url, :username, :password - end - - class BuildDetails < Build - expose :commands - expose :repo_url - expose :before_sha - expose :allow_git_fetch - expose :token - expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } - - expose :options do |model| - # This part ensures that output of old API is still the same after adding support - # for extended docker configuration options, used by new API - # - # I'm leaving this here, not in the model, because it should be removed at the same time - # when old API will be removed (planned for August 2017). - model.options.dup.tap do |options| - options[:image] = options[:image][:name] if options[:image].is_a?(Hash) - options[:services]&.map! do |service| - if service.is_a?(Hash) - service[:name] - else - service - end - end - end - end - - expose :timeout do |model| - model.timeout - end - - expose :variables - expose :depends_on_builds, using: Build - - expose :credentials, using: BuildCredentials - end - - class Runner < Grape::Entity - expose :id, :token - end - - class RunnerProject < Grape::Entity - expose :id, :project_id, :runner_id - end - - class WebHook < Grape::Entity - expose :id, :project_id, :url - end - - class TriggerRequest < Grape::Entity - expose :id, :variables - expose :pipeline, using: Commit, as: :commit - end - end - end -end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb deleted file mode 100644 index a40b6ab6c9f..00000000000 --- a/lib/ci/api/helpers.rb +++ /dev/null @@ -1,89 +0,0 @@ -module Ci - module API - module Helpers - BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze - BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 10 * 60 - - def authenticate_runners! - forbidden! unless runner_registration_token_valid? - end - - def authenticate_runner! - forbidden! unless current_runner - end - - def authenticate_build! - build = Ci::Build.find_by_id(params[:id]) - - validate_build!(build) do - forbidden! unless build_token_valid?(build) - end - - build - end - - def validate_build!(build) - not_found! unless build - - yield if block_given? - - project = build.project - forbidden!('Project has been deleted!') if project.nil? || project.pending_delete? - forbidden!('Build has been erased!') if build.erased? - end - - def runner_registration_token_valid? - ActiveSupport::SecurityUtils.variable_size_secure_compare( - params[:token], - current_application_settings.runners_registration_token) - end - - def build_token_valid?(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - - # We require to also check `runners_token` to maintain compatibility with old version of runners - token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) - end - - def update_runner_info - return unless update_runner? - - current_runner.contacted_at = Time.now - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - - def update_runner? - # Use a random threshold to prevent beating DB updates. - # It generates a distribution between [40m, 80m]. - # - contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - - current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= contacted_at_max_age - end - - def build_not_found! - if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? / - no_content! - else - not_found! - end - end - - def current_runner - @runner ||= Runner.find_by_token(params[:token].to_s) - end - - def get_runner_version_from_params - return unless params["info"].present? - attributes_for_keys(%w(name version revision platform architecture), params["info"]) - end - - def max_artifacts_size - current_application_settings.max_artifacts_size.megabytes.to_i - end - end - end -end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb deleted file mode 100644 index 45aa2adccf5..00000000000 --- a/lib/ci/api/runners.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Ci - module API - class Runners < Grape::API - resource :runners do - desc 'Delete a runner' - params do - requires :token, type: String, desc: 'The unique token of the runner' - end - delete "delete" do - authenticate_runner! - - status(200) - Ci::Runner.find_by_token(params[:token]).destroy - end - - desc 'Register a new runner' do - success Entities::Runner - end - params do - requires :token, type: String, desc: 'The unique token of the runner' - optional :description, type: String, desc: 'The description of the runner' - optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for' - optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs' - optional :locked, type: Boolean, desc: 'Lock this runner for this specific project' - end - post "register" do - runner_params = declared(params, include_missing: false).except(:token) - - runner = - if runner_registration_token_valid? - # Create shared runner. Requires admin access - Ci::Runner.create(runner_params.merge(is_shared: true)) - elsif project = Project.find_by(runners_token: params[:token]) - # Create a specific runner for project. - project.runners.create(runner_params) - end - - return forbidden! unless runner - - if runner.id - runner.update(get_runner_version_from_params) - present runner, with: Entities::Runner - else - not_found! - end - end - end - end - end -end diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb deleted file mode 100644 index 6225203f223..00000000000 --- a/lib/ci/api/triggers.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Ci - module API - class Triggers < Grape::API - resource :projects do - desc 'Trigger a GitLab CI project build' do - success Entities::TriggerRequest - end - params do - requires :id, type: Integer, desc: 'The ID of a CI project' - requires :ref, type: String, desc: "The name of project's branch or tag" - requires :token, type: String, desc: 'The unique token of the trigger' - optional :variables, type: Hash, desc: 'Optional build variables' - end - post ":id/refs/:ref/trigger" do - project = Project.find_by(ci_id: params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token]) - not_found! unless project && trigger - unauthorized! unless trigger.project == project - - # Validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - # create request and trigger builds - result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref], variables) - pipeline = result.pipeline - - if pipeline.persisted? - present result.trigger_request, with: Entities::TriggerRequest - else - render_validation_error!(pipeline) - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb new file mode 100644 index 00000000000..b1ff0900709 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -0,0 +1,77 @@ +module Gitlab + module BackgroundMigration + class MigrateStageStatus + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + scope :latest, -> { where(retried: [false, nil]) } + scope :created, -> { where(status: 'created') } + scope :running, -> { where(status: 'running') } + scope :pending, -> { where(status: 'pending') } + scope :success, -> { where(status: 'success') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + where("allow_failure = ? OR status IN (?)", + false, %w[created pending running success skipped]) + end + + def self.status_sql + scope_relevant = latest.exclude_ignored + scope_warnings = latest.failed_but_allowed + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql + + <<-SQL.strip_heredoc + (CASE + WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} + WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} + WHEN (#{created}) > 0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} + END) + SQL + end + end + + def perform(start_id, stop_id) + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql + + sql = <<-SQL + UPDATE ci_stages SET status = (#{status_sql}) + WHERE ci_stages.status IS NULL + AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + SQL + + ActiveRecord::Base.connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 53aa5b12489..eb3731ba35a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -64,7 +64,6 @@ module Gitlab end delegate :empty?, - :bare?, to: :rugged delegate :exists?, to: :gitaly_repository_client @@ -126,6 +125,8 @@ module Gitlab # This is to work around a bug in libgit2 that causes in-memory refs to # be stale/invalid when packed-refs is changed. # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/474 def find_branch(name, force_reload = false) reload_rugged if force_reload @@ -231,10 +232,6 @@ module Gitlab branch_names + tag_names end - def has_commits? - !empty? - end - # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -574,6 +571,8 @@ module Gitlab end # Delete the specified branch from the repository + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def delete_branch(branch_name) rugged.branches.delete(branch_name) end @@ -583,6 +582,8 @@ module Gitlab # Examples: # create_branch("feature") # create_branch("other-feature", "master") + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) @@ -592,38 +593,26 @@ module Gitlab raise InvalidRef.new("Invalid reference #{start_point}") end - # Return an array of this repository's remote names - def remote_names - rugged.remotes.each_name.to_a - end - # Delete the specified remote from this repository. def remote_delete(remote_name) rugged.remotes.delete(remote_name) + nil end - # Add a new remote to this repository. Returns a Rugged::Remote object + # Add a new remote to this repository. def remote_add(remote_name, url) rugged.remotes.create(remote_name, url) + nil end # Update the specified remote using the values in the +options+ hash # # Example # repo.update_remote("origin", url: "path/to/repo") - def remote_update(remote_name, options = {}) + def remote_update(remote_name, url:) # TODO: Implement other remote options - rugged.remotes.set_url(remote_name, options[:url]) if options[:url] - end - - # Fetch the specified remote - def fetch(remote_name) - rugged.remotes[remote_name].fetch - end - - # Push +*refspecs+ to the remote identified by +remote_name+. - def push(remote_name, *refspecs) - rugged.remotes[remote_name].push(refspecs) + rugged.remotes.set_url(remote_name, url) + nil end AUTOCRLF_VALUES = { diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index b36e81278d6..2d58fb0186e 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -80,8 +80,8 @@ module Gitlab def tree_entries(repository, revision, path) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, - revision: revision, - path: path.presence || '.' + revision: GitalyClient.encode(revision), + path: path.present? ? GitalyClient.encode(path) : '.' ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9d9ebcb389a..894950e341f 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -98,6 +98,7 @@ excluded_attributes: - :last_activity_at - :last_repository_updated_at - :last_repository_check_at + - :storage_version snippets: - :expired_at merge_request_diff: diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 208f0e1bbea..4d6bbda15f3 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -1,12 +1,31 @@ module Gitlab # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + # + # Its use requires the cooperation of the sidekiq jobs themselves. Set up the + # waiter, then start the jobs, passing them its `key`. Their `perform` methods + # should look like: + # + # def perform(args, notify_key) + # # do work + # ensure + # ::Gitlab::JobWaiter.notify(notify_key, jid) + # end + # + # The JobWaiter blocks popping items from a Redis array. All the sidekiq jobs + # push to that array when done. Once the waiter has popped `count` items, it + # knows all the jobs are done. class JobWaiter - # The sleep interval between checking keys, in seconds. - INTERVAL = 0.1 + def self.notify(key, jid) + Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } + end + + attr_reader :key, :jobs_remaining, :finished - # jobs - The job IDs to wait for. - def initialize(jobs) - @jobs = jobs + # jobs_remaining - the number of jobs left to wait for + def initialize(jobs_remaining) + @key = "gitlab:job_waiter:#{SecureRandom.uuid}" + @jobs_remaining = jobs_remaining + @finished = [] end # Waits for all the jobs to be completed. @@ -15,13 +34,33 @@ module Gitlab # ensures we don't indefinitely block a caller in case a job takes # long to process, or is never processed. def wait(timeout = 10) - start = Time.current + deadline = Time.now.utc + timeout + + Gitlab::Redis::SharedState.with do |redis| + # Fallback key expiry: allow a long grace period to reduce the chance of + # a job pushing to an expired key and recreating it + redis.expire(key, [timeout * 2, 10.minutes.to_i].max) + + while jobs_remaining > 0 + # Redis will not take fractional seconds. Prefer waiting too long over + # not waiting long enough + seconds_left = (deadline - Time.now.utc).ceil - while (Time.current - start) <= timeout - break if SidekiqStatus.all_completed?(@jobs) + # Redis interprets 0 as "wait forever", so skip the final `blpop` call + break if seconds_left <= 0 - sleep(INTERVAL) # to not overload Redis too much. + list, jid = redis.blpop(key, timeout: seconds_left) + break unless list && jid # timed out + + @finished << jid + @jobs_remaining -= 1 + end + + # All jobs have finished, so expire the key immediately + redis.expire(key, 0) if jobs_remaining == 0 end + + finished end end end diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb index d4d39a888e7..5512afa45a8 100644 --- a/lib/gitlab/sidekiq_throttler.rb +++ b/lib/gitlab/sidekiq_throttler.rb @@ -3,6 +3,8 @@ module Gitlab class << self def execute! if Gitlab::CurrentSettings.sidekiq_throttling_enabled? + require 'sidekiq-limit_fetch' + Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| Sidekiq::Queue[queue].limit = queue_limit end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 48bd9139ce8..6e10ba374bf 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -11,6 +11,12 @@ namespace :gitlab do # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + if Project.current_application_settings.hashed_storage_enabled + puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red) + + exit 1 + end + Gitlab.config.repositories.storages.each_value do |repository_storage| git_base_path = repository_storage['path'] repos_to_import = Dir.glob(git_base_path + '/**/*.git') |