diff options
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/repository_import_worker.rb | 4 | ||||
-rw-r--r-- | app/workers/stuck_import_jobs_worker.rb | 37 |
2 files changed, 40 insertions, 1 deletions
diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 68a6fd76e70..b33ba2ed7c1 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -2,6 +2,8 @@ class RepositoryImportWorker include Sidekiq::Worker include DedicatedSidekiqQueue + sidekiq_options status_expiration: StuckImportJobsWorker::IMPORT_EXPIRATION + attr_accessor :project, :current_user def perform(project_id) @@ -12,7 +14,7 @@ class RepositoryImportWorker import_url: @project.import_url, path: @project.path_with_namespace) - project.update_column(:import_error, nil) + project.update_columns(import_jid: self.jid, import_error: nil) result = Projects::ImportService.new(project, current_user).execute diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb new file mode 100644 index 00000000000..bfc5e667bb6 --- /dev/null +++ b/app/workers/stuck_import_jobs_worker.rb @@ -0,0 +1,37 @@ +class StuckImportJobsWorker + include Sidekiq::Worker + include CronjobQueue + + IMPORT_EXPIRATION = 15.hours.to_i + + def perform + stuck_projects.find_in_batches(batch_size: 500) do |group| + jids = group.map(&:import_jid) + + # Find the jobs that aren't currently running or that exceeded the threshold. + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids) + + if completed_jids.any? + completed_ids = group.select { |project| completed_jids.include?(project.import_jid) }.map(&:id) + + fail_batch!(completed_jids, completed_ids) + end + end + end + + private + + def stuck_projects + Project.select('id, import_jid').with_import_status(:started).where.not(import_jid: nil) + end + + def fail_batch!(completed_jids, completed_ids) + Project.where(id: completed_ids).update_all(import_status: 'failed', import_error: error_message) + + Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.join(', ')}") + end + + def error_message + "Import timed out. Import took longer than #{IMPORT_EXPIRATION} seconds" + end +end |