From b6db9a4d78ae873d0e94f0070ca26ae5e6742ebe Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 19 Jul 2019 18:02:32 +0700 Subject: Use ArchiveTraceWorker in ArchiveTracesCronWorker Currently, ArchiveTracesCronWorker keeps respawing repeatedly in a short interval because it's killed by Sidekiq Memory Killer. We split the memory consumption into multiple sidekiq jobs in order to avoid the unstable performance. --- app/workers/ci/archive_traces_cron_worker.rb | 13 ++++++++++++- .../unreleased/use-archive-trace-worker-in-cron-worker.yml | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/use-archive-trace-worker-in-cron-worker.yml diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb index f65ff239866..f0eb2cb16e0 100644 --- a/app/workers/ci/archive_traces_cron_worker.rb +++ b/app/workers/ci/archive_traces_cron_worker.rb @@ -11,7 +11,18 @@ module Ci # This could happen when ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL # More details in https://gitlab.com/gitlab-org/gitlab-ce/issues/36791 Ci::Build.finished.with_live_trace.find_each(batch_size: 100) do |build| - Ci::ArchiveTraceService.new.execute(build) + if Feature.enabled?(:ci_archive_trace_async) + ArchiveTraceWorker.perform_async(build.id) + else + begin + Ci::ArchiveTraceService.new.execute(build) + ensure + ## + # This is the temporary solution for avoiding the memory bloat. + # See more https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667#note_192401907 + GC.start if Feature.enabled?(:ci_archive_trace_force_gc, default_enabled: true) + end + end end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/changelogs/unreleased/use-archive-trace-worker-in-cron-worker.yml b/changelogs/unreleased/use-archive-trace-worker-in-cron-worker.yml new file mode 100644 index 00000000000..c5c1d210b34 --- /dev/null +++ b/changelogs/unreleased/use-archive-trace-worker-in-cron-worker.yml @@ -0,0 +1,6 @@ +--- +title: Fix ArchiveTracesCronWorker respawning in a short interval by sidekiq memory + killer +merge_request: 30940 +author: +type: fixed -- cgit v1.2.1