Merge branch 'add-counter-for-trace-chunks' into 'master'

Improve traceability for failed attempts of archiving traces

Closes #51502

See merge request gitlab-org/gitlab-ce!21826

3 files changed, 37 insertions, 14 deletions

diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb
new file mode 100644
index 00000000000..a1dd00721b5
--- /dev/null
+++ b/app/services/ci/archive_trace_service.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Ci
+  class ArchiveTraceService
+    def execute(job)
+      job.trace.archive!
+    rescue ::Gitlab::Ci::Trace::AlreadyArchivedError
+      # It's already archived, thus we can safely ignore this exception.
+    rescue => e
+      # Tracks this error with application logs, Sentry, and Prometheus.
+      # If `archive!` keeps failing for over a week, that could incur data loss.
+      # (See more https://docs.gitlab.com/ee/administration/job_traces.html#new-live-trace-architecture)
+      # In order to avoid interrupting the system, we do not raise an exception here.
+      archive_error(e, job)
+    end
+
+    private
+
+    def failed_archive_counter
+      @failed_archive_counter ||=
+        Gitlab::Metrics.counter(:job_trace_archive_failed_total,
+                                "Counter of failed attempts of trace archiving")
+    end
+
+    def archive_error(error, job)
+      failed_archive_counter.increment
+      Rails.logger.error "Failed to archive trace. id: #{job.id} message: #{error.message}"
+
+      Gitlab::Sentry
+        .track_exception(error,
+                         issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/51502',
+                         extra: { job_id: job.id })
+    end
+  end
+end
diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb
index c1283e9b2fc..4a9becf0ca7 100644
--- a/app/workers/archive_trace_worker.rb
+++ b/app/workers/archive_trace_worker.rb
@@ -7,7 +7,7 @@ class ArchiveTraceWorker
   # rubocop: disable CodeReuse/ActiveRecord
   def perform(job_id)
     Ci::Build.without_archived_trace.find_by(id: job_id).try do |job|
-      job.trace.archive!
+      Ci::ArchiveTraceService.new.execute(job)
     end
   end
   # rubocop: enable CodeReuse/ActiveRecord
diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb
index 7443aad1380..f65ff239866 100644
--- a/app/workers/ci/archive_traces_cron_worker.rb
+++ b/app/workers/ci/archive_traces_cron_worker.rb
@@ -11,21 +11,9 @@ module Ci
       # This could happen when ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL
       # More details in https://gitlab.com/gitlab-org/gitlab-ce/issues/36791
       Ci::Build.finished.with_live_trace.find_each(batch_size: 100) do |build|
-        begin
-          build.trace.archive!
-        rescue ::Gitlab::Ci::Trace::AlreadyArchivedError
-        rescue => e
-          failed_archive_counter.increment
-          Rails.logger.error "Failed to archive stale live trace. id: #{build.id} message: #{e.message}"
-        end
+        Ci::ArchiveTraceService.new.execute(build)
       end
     end
     # rubocop: enable CodeReuse/ActiveRecord
-
-    private
-
-    def failed_archive_counter
-      @failed_archive_counter ||= Gitlab::Metrics.counter(:job_trace_archive_failed_total, "Counter of failed attempts of traces archiving")
-    end
   end
 end