diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2019-08-13 15:14:33 +0200 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2019-08-14 11:34:44 +0200 |
commit | cee2f86d5724fcb073a8abdfbaf83869a8de85f1 (patch) | |
tree | 6dbc36417f1572caeca68aaa9ba91fc34a93de31 | |
parent | a55869483d023978655658d389aad36d63c9d2b2 (diff) | |
download | gitlab-ce-cee2f86d5724fcb073a8abdfbaf83869a8de85f1.tar.gz |
Optimise DAG processingoptimise-dag-processing
-rw-r--r-- | app/models/commit_status.rb | 21 | ||||
-rw-r--r-- | app/models/concerns/has_status.rb | 5 | ||||
-rw-r--r-- | app/services/ci/process_pipeline_service.rb | 27 |
3 files changed, 37 insertions, 16 deletions
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index a88cac6b8e6..4be4d95b4a1 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -40,8 +40,11 @@ class CommitStatus < ApplicationRecord scope :ordered, -> { order(:name) } scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) } scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } + scope :before_stage, -> (index) { where('stage_idx < ?', index) } + scope :for_stage, -> (index) { where(stage_idx: index) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) } + scope :for_ids, -> (ids) { where(id: ids) } scope :with_needs, -> (names = nil) do needs = Ci::BuildNeed.scoped_build.select(1) @@ -49,8 +52,10 @@ class CommitStatus < ApplicationRecord where('EXISTS (?)', needs).preload(:needs) end - scope :without_needs, -> do - where('NOT EXISTS (?)', Ci::BuildNeed.scoped_build.select(1)) + scope :without_needs, -> (names = nil) do + needs = Ci::BuildNeed.scoped_build.select(1) + needs = needs.where(name: names) if names + where('NOT EXISTS (?)', needs) end # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily @@ -149,6 +154,18 @@ class CommitStatus < ApplicationRecord end end + def self.names + select(:name) + end + + def self.status_for_prior_stages(index) + before_stage(index).latest.status || 'success' + end + + def self.status_for_names(names) + where(name: names).latest.status || 'success' + end + def locking_enabled? will_save_change_to_status? end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 27a5c3d5286..71ebb586c13 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -106,10 +106,15 @@ module HasStatus scope :running_or_pending, -> { with_status(:running, :pending) } scope :finished, -> { with_status(:success, :failed, :canceled) } scope :failed_or_canceled, -> { with_status(:failed, :canceled) } + scope :incomplete, -> { without_statuses(completed_statuses) } scope :cancelable, -> do where(status: [:running, :preparing, :pending, :created, :scheduled]) end + + scope :without_statuses, -> (names) do + with_status(all_state_names - names.to_a) + end end def started? diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 99d4ff9ecd1..f4bd457ebc6 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -42,14 +42,19 @@ module Ci return false unless trigger_build_ids.present? return false unless Feature.enabled?(:ci_dag_support, project) - # rubocop: disable CodeReuse/ActiveRecord - trigger_build_names = pipeline.statuses - .where(id: trigger_build_ids) - .select(:name) - # rubocop: enable CodeReuse/ActiveRecord + # we find processables that are dependent: + # 1. because of current dependency, + trigger_build_names = pipeline.processables.latest + .for_ids(trigger_build_ids).names + # 2. does not have builds that not yet complete + incomplete_build_names = pipeline.processables.latest + .incomplete.names + + # Each found processable is guaranteed here to have completed status created_processables .with_needs(trigger_build_names) + .without_needs(incomplete_build_names) .find_each .map(&method(:process_build_with_needs)) .any? @@ -70,17 +75,13 @@ module Ci end end - # rubocop: disable CodeReuse/ActiveRecord def status_for_prior_stages(index) - pipeline.builds.where('stage_idx < ?', index).latest.status || 'success' + pipeline.processables.status_for_prior_stages(index) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def status_for_build_needs(needs) - pipeline.builds.where(name: needs).latest.status || 'success' + pipeline.processables.status_for_names(needs) end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def stage_indexes_of_created_processables_without_needs @@ -89,12 +90,10 @@ module Ci end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def created_processables_in_stage_without_needs(index) created_processables_without_needs - .where(stage_idx: index) + .for_stage(index) end - # rubocop: enable CodeReuse/ActiveRecord def created_processables_without_needs if Feature.enabled?(:ci_dag_support, project) |