summaryrefslogtreecommitdiff
path: root/app/workers
diff options
context:
space:
mode:
Diffstat (limited to 'app/workers')
-rw-r--r--app/workers/repository_import_worker.rb4
-rw-r--r--app/workers/stuck_import_jobs_worker.rb37
2 files changed, 40 insertions, 1 deletions
diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb
index 68a6fd76e70..b33ba2ed7c1 100644
--- a/app/workers/repository_import_worker.rb
+++ b/app/workers/repository_import_worker.rb
@@ -2,6 +2,8 @@ class RepositoryImportWorker
include Sidekiq::Worker
include DedicatedSidekiqQueue
+ sidekiq_options status_expiration: StuckImportJobsWorker::IMPORT_EXPIRATION
+
attr_accessor :project, :current_user
def perform(project_id)
@@ -12,7 +14,7 @@ class RepositoryImportWorker
import_url: @project.import_url,
path: @project.path_with_namespace)
- project.update_column(:import_error, nil)
+ project.update_columns(import_jid: self.jid, import_error: nil)
result = Projects::ImportService.new(project, current_user).execute
diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb
new file mode 100644
index 00000000000..bfc5e667bb6
--- /dev/null
+++ b/app/workers/stuck_import_jobs_worker.rb
@@ -0,0 +1,37 @@
+class StuckImportJobsWorker
+ include Sidekiq::Worker
+ include CronjobQueue
+
+ IMPORT_EXPIRATION = 15.hours.to_i
+
+ def perform
+ stuck_projects.find_in_batches(batch_size: 500) do |group|
+ jids = group.map(&:import_jid)
+
+ # Find the jobs that aren't currently running or that exceeded the threshold.
+ completed_jids = Gitlab::SidekiqStatus.completed_jids(jids)
+
+ if completed_jids.any?
+ completed_ids = group.select { |project| completed_jids.include?(project.import_jid) }.map(&:id)
+
+ fail_batch!(completed_jids, completed_ids)
+ end
+ end
+ end
+
+ private
+
+ def stuck_projects
+ Project.select('id, import_jid').with_import_status(:started).where.not(import_jid: nil)
+ end
+
+ def fail_batch!(completed_jids, completed_ids)
+ Project.where(id: completed_ids).update_all(import_status: 'failed', import_error: error_message)
+
+ Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.join(', ')}")
+ end
+
+ def error_message
+ "Import timed out. Import took longer than #{IMPORT_EXPIRATION} seconds"
+ end
+end