diff options
| -rw-r--r-- | app/models/ci/build.rb | 5 | ||||
| -rw-r--r-- | db/post_migrate/20180529152628_archive_legacy_traces.rb | 19 | ||||
| -rw-r--r-- | lib/gitlab/background_migration/archive_legacy_traces.rb | 77 | ||||
| -rw-r--r-- | lib/tasks/gitlab/traces.rake | 4 | ||||
| -rw-r--r-- | spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb | 2 |
5 files changed, 22 insertions, 85 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 75fd55a8f7b..6cbf9108244 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -55,6 +55,11 @@ module Ci where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive) end + + scope :without_archived_trace, ->() do + where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace) + end + scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } diff --git a/db/post_migrate/20180529152628_archive_legacy_traces.rb b/db/post_migrate/20180529152628_archive_legacy_traces.rb index 78ec1ab1a94..12f219f606c 100644 --- a/db/post_migrate/20180529152628_archive_legacy_traces.rb +++ b/db/post_migrate/20180529152628_archive_legacy_traces.rb @@ -2,7 +2,7 @@ class ArchiveLegacyTraces < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false - BATCH_SIZE = 10_000 + BATCH_SIZE = 5000 BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces' disable_ddl_transaction! @@ -14,25 +14,14 @@ class ArchiveLegacyTraces < ActiveRecord::Migration scope :finished, -> { where(status: [:success, :failed, :canceled]) } - scope :without_new_traces, ->() do - where('NOT EXISTS (?)', - ::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + scope :without_archived_trace, -> do + where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)') end end - class JobArtifact < ActiveRecord::Base - self.table_name = 'ci_job_artifacts' - - enum file_type: { - archive: 1, - metadata: 2, - trace: 3 - } - end - def up queue_background_migration_jobs_by_range_at_intervals( - ::ArchiveLegacyTraces::Build.finished.without_new_traces, + ::ArchiveLegacyTraces::Build.finished.without_archived_trace, BACKGROUND_MIGRATION_CLASS, 5.minutes, batch_size: BATCH_SIZE) diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 9741a7c181e..0999ea95e56 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -5,73 +5,18 @@ module Gitlab module BackgroundMigration class ArchiveLegacyTraces - class Build < ActiveRecord::Base - include ::HasStatus - - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled # Disable STI - - belongs_to :project, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Project' - has_one :job_artifacts_trace, -> () { where(file_type: ArchiveLegacyTraces::JobArtifact.file_types[:trace]) }, class_name: 'ArchiveLegacyTraces::JobArtifact', foreign_key: :job_id - has_many :trace_chunks, foreign_key: :build_id, class_name: 'ArchiveLegacyTraces::BuildTraceChunk' - - scope :finished, -> { where(status: [:success, :failed, :canceled]) } - - scope :without_new_traces, ->() do - finished.where('NOT EXISTS (?)', - BackgroundMigration::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) - end - - def trace - ::Gitlab::Ci::Trace.new(self) - end - - def trace=(data) - raise NotImplementedError - end - - def old_trace - read_attribute(:trace) - end - - def erase_old_trace! - update_column(:trace, nil) - end - end - - class JobArtifact < ActiveRecord::Base - self.table_name = 'ci_job_artifacts' - - belongs_to :build - belongs_to :project - - mount_uploader :file, JobArtifactUploader - - enum file_type: { - archive: 1, - metadata: 2, - trace: 3 - } - end - - class BuildTraceChunk < ActiveRecord::Base - self.table_name = 'ci_build_trace_chunks' - - belongs_to :build - end - - class Project < ActiveRecord::Base - self.table_name = 'projects' - - has_many :builds, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Build' - end - def perform(start_id, stop_id) - BackgroundMigration::ArchiveLegacyTraces::Build - .finished - .without_new_traces - .where(id: (start_id..stop_id)).find_each do |build| - build.trace.archive! + # This background migrations directly refer ::Ci::Build model which is defined in application code. + # In general, migration code should be isolated as much as possible in order to be idempotent. + # However, `archive!` logic is too complicated to be replicated. So we chose a way to refer ::Ci::Build directly + # and we don't change the `archive!` logic until 11.1 + ::Ci::Build.finished.without_archived_trace + .where(id: start_id..stop_id).find_each do |build| + begin + build.trace.archive! + rescue => e + Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" + end end end end diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake index fd2a4f2d11a..ddcca69711f 100644 --- a/lib/tasks/gitlab/traces.rake +++ b/lib/tasks/gitlab/traces.rake @@ -8,9 +8,7 @@ namespace :gitlab do logger = Logger.new(STDOUT) logger.info('Archiving legacy traces') - Ci::Build.finished - .where('NOT EXISTS (?)', - Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + Ci::Build.finished.without_archived_trace .order(id: :asc) .find_in_batches(batch_size: 1000) do |jobs| job_ids = jobs.map { |job| [job.id] } diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb index ecc6eea9284..1a62b30ce81 100644 --- a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb +++ b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb @@ -14,7 +14,7 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2 @legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, build.created_at.utc.strftime("%Y_%m"), build.project_id.to_s) - + FileUtils.mkdir_p(@legacy_trace_dir) @legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log") |