From 58371efbb0bd051d3a82f82acac98ad4692efeb4 Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Fri, 24 Mar 2017 12:08:34 +0100
Subject: Periodically mark projects that are stuck in importing as failed

Adds import jid to projects
Refactor middleware to set custom expiration time via sidekiq options
Add completed_jids option to sidekiq status and a few other changes
---
 app/workers/repository_import_worker.rb |  4 ++-
 app/workers/stuck_import_jobs_worker.rb | 50 +++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100644 app/workers/stuck_import_jobs_worker.rb

(limited to 'app/workers')

diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb
index 68a6fd76e70..b33ba2ed7c1 100644
--- a/app/workers/repository_import_worker.rb
+++ b/app/workers/repository_import_worker.rb
@@ -2,6 +2,8 @@ class RepositoryImportWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
+  sidekiq_options status_expiration: StuckImportJobsWorker::IMPORT_EXPIRATION
+
   attr_accessor :project, :current_user
 
   def perform(project_id)
@@ -12,7 +14,7 @@ class RepositoryImportWorker
                               import_url: @project.import_url,
                               path: @project.path_with_namespace)
 
-    project.update_column(:import_error, nil)
+    project.update_columns(import_jid: self.jid, import_error: nil)
 
     result = Projects::ImportService.new(project, current_user).execute
 
diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb
new file mode 100644
index 00000000000..c7871d99492
--- /dev/null
+++ b/app/workers/stuck_import_jobs_worker.rb
@@ -0,0 +1,50 @@
+class StuckImportJobsWorker
+  include Sidekiq::Worker
+  include CronjobQueue
+
+  EXCLUSIVE_LEASE_KEY = 'fail_stuck_imports_worker_lease'.freeze
+  IMPORT_EXPIRATION = 15.hours.to_i
+
+  def perform
+    return unless try_obtain_lease
+
+    stuck_projects.find_in_batches(batch_size: 500) do |group|
+      jids = group.map(&:import_jid)
+
+      # Find the jobs that aren't currently running or that exceeded the threshold.
+      completed_jids = Gitlab::SidekiqStatus.completed_jids(jids)
+
+      if completed_jids.any?
+        completed_ids = group.select { |project| completed_jids.include?(project.import_jid) }.map(&:id)
+
+        fail_batch!(completed_jids, completed_ids)
+      end
+    end
+
+    remove_lease
+  end
+
+  private
+
+  def stuck_projects
+    Project.select('id, import_jid').with_import_status(:started).where.not(import_jid: nil)
+  end
+
+  def fail_batch!(completed_jids, completed_ids)
+    Project.where(id: completed_ids).update_all(import_status: 'failed', import_error: error_message)
+
+    Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.join(', ')}")
+  end
+
+  def error_message
+    "Import timed out. Import took longer than #{IMPORT_EXPIRATION} seconds"
+  end
+
+  def try_obtain_lease
+    @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain
+  end
+
+  def remove_lease
+    Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
+  end
+end
-- 
cgit v1.2.1


From 94716c279f35c787d5b41ed626ad9bfc1acf83aa Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Thu, 6 Apr 2017 09:48:58 +0200
Subject: remove unnecessary lease as cron job

---
 app/workers/stuck_import_jobs_worker.rb | 13 -------------
 1 file changed, 13 deletions(-)

(limited to 'app/workers')

diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb
index c7871d99492..bfc5e667bb6 100644
--- a/app/workers/stuck_import_jobs_worker.rb
+++ b/app/workers/stuck_import_jobs_worker.rb
@@ -2,12 +2,9 @@ class StuckImportJobsWorker
   include Sidekiq::Worker
   include CronjobQueue
 
-  EXCLUSIVE_LEASE_KEY = 'fail_stuck_imports_worker_lease'.freeze
   IMPORT_EXPIRATION = 15.hours.to_i
 
   def perform
-    return unless try_obtain_lease
-
     stuck_projects.find_in_batches(batch_size: 500) do |group|
       jids = group.map(&:import_jid)
 
@@ -20,8 +17,6 @@ class StuckImportJobsWorker
         fail_batch!(completed_jids, completed_ids)
       end
     end
-
-    remove_lease
   end
 
   private
@@ -39,12 +34,4 @@ class StuckImportJobsWorker
   def error_message
     "Import timed out. Import took longer than #{IMPORT_EXPIRATION} seconds"
   end
-
-  def try_obtain_lease
-    @uuid = Gitlab::ExclusiveLease.new(EXCLUSIVE_LEASE_KEY, timeout: 30.minutes).try_obtain
-  end
-
-  def remove_lease
-    Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
-  end
 end
-- 
cgit v1.2.1