From e7ee84aad4237eaa16f2aba75b4d2c7860625c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 1 Aug 2019 14:26:49 +0000 Subject: Add support for DAG This implements the support for `needs:` keyword as part of GitLab CI. That makes some of the jobs to be run out of order. --- app/models/commit_status.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'app/models/commit_status.rb') diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index be6f3e9c5b0..d7eb78db5b8 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -43,6 +43,12 @@ class CommitStatus < ApplicationRecord scope :after_stage, -> (index) { where('stage_idx > ?', index) } scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) } + scope :with_needs, -> (names = nil) do + needs = Ci::BuildNeed.scoped_build.select(1) + needs = needs.where(name: names) if names + where('EXISTS (?)', needs).preload(:needs) + end + # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. enum_with_nil failure_reason: ::CommitStatusEnums.failure_reasons @@ -116,7 +122,7 @@ class CommitStatus < ApplicationRecord commit_status.run_after_commit do if pipeline_id if complete? || manual? - PipelineProcessWorker.perform_async(pipeline_id) + BuildProcessWorker.perform_async(id) else PipelineUpdateWorker.perform_async(pipeline_id) end -- cgit v1.2.1 From a2bbf7b8c197922067d1266249197e5f60e0fd7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Fri, 2 Aug 2019 18:12:24 +0200 Subject: Properly process `needs:` with `when:` Currently, some of the jobs with `needs:` would be processed in stages, it means that `when:` for such jobs would not be respected. This changes the behavior to have a separate execution paths for jobs with `needs:`. --- app/models/commit_status.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models/commit_status.rb') diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index d7eb78db5b8..a9c29fb390b 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -49,6 +49,10 @@ class CommitStatus < ApplicationRecord where('EXISTS (?)', needs).preload(:needs) end + scope :without_needs, -> do + where('NOT EXISTS (?)', Ci::BuildNeed.scoped_build.select(1)) + end + # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. enum_with_nil failure_reason: ::CommitStatusEnums.failure_reasons -- cgit v1.2.1 From 593490e5ac6ec5a92fdcc9b82eebbb455ea6cf48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Fri, 2 Aug 2019 19:31:01 +0200 Subject: Extend PipelineProcessWorker to accept a list of builds This changes used worker from `BuildProcessWorker` to `PipelineProcessWorker` to make pipeline processing much simpler. We process `pipeline_id`, based on some triggers. --- app/models/commit_status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models/commit_status.rb') diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index d7eb78db5b8..004ff037558 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -122,7 +122,7 @@ class CommitStatus < ApplicationRecord commit_status.run_after_commit do if pipeline_id if complete? || manual? - BuildProcessWorker.perform_async(id) + PipelineProcessWorker.perform_async(pipeline_id, [id]) else PipelineUpdateWorker.perform_async(pipeline_id) end -- cgit v1.2.1 From cee2f86d5724fcb073a8abdfbaf83869a8de85f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 13 Aug 2019 15:14:33 +0200 Subject: Optimise DAG processing --- app/models/commit_status.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'app/models/commit_status.rb') diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index a88cac6b8e6..4be4d95b4a1 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -40,8 +40,11 @@ class CommitStatus < ApplicationRecord scope :ordered, -> { order(:name) } scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) } scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } + scope :before_stage, -> (index) { where('stage_idx < ?', index) } + scope :for_stage, -> (index) { where(stage_idx: index) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) } + scope :for_ids, -> (ids) { where(id: ids) } scope :with_needs, -> (names = nil) do needs = Ci::BuildNeed.scoped_build.select(1) @@ -49,8 +52,10 @@ class CommitStatus < ApplicationRecord where('EXISTS (?)', needs).preload(:needs) end - scope :without_needs, -> do - where('NOT EXISTS (?)', Ci::BuildNeed.scoped_build.select(1)) + scope :without_needs, -> (names = nil) do + needs = Ci::BuildNeed.scoped_build.select(1) + needs = needs.where(name: names) if names + where('NOT EXISTS (?)', needs) end # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily @@ -149,6 +154,18 @@ class CommitStatus < ApplicationRecord end end + def self.names + select(:name) + end + + def self.status_for_prior_stages(index) + before_stage(index).latest.status || 'success' + end + + def self.status_for_names(names) + where(name: names).latest.status || 'success' + end + def locking_enabled? will_save_change_to_status? end -- cgit v1.2.1