From 3248ca0467a4bd816f281db1b30ca83df2865edd Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 24 Apr 2018 17:08:43 +0900 Subject: Add per-project pipeline id --- app/models/ci/pipeline.rb | 3 +++ app/models/internal_id.rb | 2 +- .../20180424160449_add_pipeline_iid_to_ci_pipelines.rb | 15 +++++++++++++++ db/schema.rb | 1 + 4 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index e1b9bc76475..65b282ecec4 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -7,12 +7,15 @@ module Ci include Presentable include Gitlab::OptimisticLocking include Gitlab::Utils::StrongMemoize + include AtomicInternalId belongs_to :project, inverse_of: :pipelines belongs_to :user belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.pipelines.maximum(:iid) } + has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index 189942c5ad8..dbd82dda06e 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -14,7 +14,7 @@ class InternalId < ActiveRecord::Base belongs_to :project belongs_to :namespace - enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4 } + enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4, ci_pipelines: 5 } validates :usage, presence: true diff --git a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb new file mode 100644 index 00000000000..d732116e9ce --- /dev/null +++ b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb @@ -0,0 +1,15 @@ +class AddPipelineIidToCiPipelines < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column :ci_pipelines, :iid, :integer + end + + def down + remove_column :ci_pipelines, :iid, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index 10cd1bff125..d4bc075eb2e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -434,6 +434,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do t.integer "config_source" t.boolean "protected" t.integer "failure_reason" + t.integer "iid" end add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree -- cgit v1.2.1 From 332d3d0ef5215c2a363c99c90fb7d31af90a0a5a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 25 Apr 2018 21:03:35 +0900 Subject: Change column name to iid_per_project. Add index to project_id and iid --- .../20180424160449_add_pipeline_iid_to_ci_pipelines.rb | 4 ++-- ...0180425205249_add_index_constraints_to_pipeline_iid.rb | 15 +++++++++++++++ db/schema.rb | 5 +++-- 3 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb diff --git a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb index d732116e9ce..128377e1c9d 100644 --- a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb +++ b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb @@ -6,10 +6,10 @@ class AddPipelineIidToCiPipelines < ActiveRecord::Migration disable_ddl_transaction! def up - add_column :ci_pipelines, :iid, :integer + add_column :ci_pipelines, :iid_per_project, :integer end def down - remove_column :ci_pipelines, :iid, :integer + remove_column :ci_pipelines, :iid_per_project, :integer end end diff --git a/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb b/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb new file mode 100644 index 00000000000..7daa7197b7c --- /dev/null +++ b/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb @@ -0,0 +1,15 @@ +class AddIndexConstraintsToPipelineIid < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_pipelines, [:project_id, :iid_per_project], unique: true, where: 'iid_per_project IS NOT NULL' + end + + def down + remove_concurrent_index :ci_pipelines, [:project_id, :iid_per_project] + end +end diff --git a/db/schema.rb b/db/schema.rb index d4bc075eb2e..af838e109b2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180425131009) do +ActiveRecord::Schema.define(version: 20180425205249) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -434,11 +434,12 @@ ActiveRecord::Schema.define(version: 20180425131009) do t.integer "config_source" t.boolean "protected" t.integer "failure_reason" - t.integer "iid" + t.integer "iid_per_project" end add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree + add_index "ci_pipelines", ["project_id", "iid_per_project"], name: "index_ci_pipelines_on_project_id_and_iid_per_project", unique: true, where: "(iid_per_project IS NOT NULL)", using: :btree add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree -- cgit v1.2.1 From 8192cd70ed109e9c0b7f4670234af11f206694d4 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 25 Apr 2018 21:42:46 +0900 Subject: Expose CI_PIPELINE_IID_PER_PROJECT variable --- app/models/ci/pipeline.rb | 3 ++- doc/ci/variables/README.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 65b282ecec4..3cdb86e7b55 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.pipelines.maximum(:iid) } + has_internal_id :iid_per_project, scope: :project, init: ->(s) { s&.project&.pipelines.count } has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline @@ -492,6 +492,7 @@ module Ci def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PIPELINE_ID', value: id.to_s) + .append(key: 'CI_PIPELINE_IID_PER_PROJECT', value: iid_per_project.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) end diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 38a988f4507..8eb0fc5ea49 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -60,6 +60,7 @@ future GitLab releases.** | **CI_RUNNER_REVISION** | all | 10.6 | GitLab Runner revision that is executing the current job | | **CI_RUNNER_EXECUTABLE_ARCH** | all | 10.6 | The OS/architecture of the GitLab Runner executable (note that this is not necessarily the same as the environment of the executor) | | **CI_PIPELINE_ID** | 8.10 | 0.5 | The unique id of the current pipeline that GitLab CI uses internally | +| **CI_PIPELINE_IID_PER_PROJECT** | 10.8 | all | The unique id of the current pipeline that GitLab CI uses internally (Incremented per project) | | **CI_PIPELINE_TRIGGERED** | all | all | The flag to indicate that job was [triggered] | | **CI_PIPELINE_SOURCE** | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, and `pipeline`. For pipelines created before GitLab 9.5, this will show as `unknown` | | **CI_PROJECT_DIR** | all | all | The full path where the repository is cloned and where the job is run | -- cgit v1.2.1 From 8327ad8d8edffe17870571aff1dd68a6c0bce9e7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 25 Apr 2018 21:44:05 +0900 Subject: Add changelog --- changelogs/unreleased/per-project-pipeline-iid.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/per-project-pipeline-iid.yml diff --git a/changelogs/unreleased/per-project-pipeline-iid.yml b/changelogs/unreleased/per-project-pipeline-iid.yml new file mode 100644 index 00000000000..78a513a9986 --- /dev/null +++ b/changelogs/unreleased/per-project-pipeline-iid.yml @@ -0,0 +1,5 @@ +--- +title: Add per-project pipeline id +merge_request: 18558 +author: +type: added -- cgit v1.2.1 From 787eddd55d3699c73a69c99eb66e6a4b3b63b330 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 25 Apr 2018 22:00:11 +0900 Subject: Revert column name change --- app/models/ci/pipeline.rb | 4 ++-- db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb | 4 ++-- db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb | 4 ++-- db/schema.rb | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 3cdb86e7b55..efab6f4283a 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid_per_project, scope: :project, init: ->(s) { s&.project&.pipelines.count } + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.pipelines.count } has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline @@ -492,7 +492,7 @@ module Ci def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PIPELINE_ID', value: id.to_s) - .append(key: 'CI_PIPELINE_IID_PER_PROJECT', value: iid_per_project.to_s) + .append(key: 'CI_PIPELINE_IID_PER_PROJECT', value: iid.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) end diff --git a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb index 128377e1c9d..d732116e9ce 100644 --- a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb +++ b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb @@ -6,10 +6,10 @@ class AddPipelineIidToCiPipelines < ActiveRecord::Migration disable_ddl_transaction! def up - add_column :ci_pipelines, :iid_per_project, :integer + add_column :ci_pipelines, :iid, :integer end def down - remove_column :ci_pipelines, :iid_per_project, :integer + remove_column :ci_pipelines, :iid, :integer end end diff --git a/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb b/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb index 7daa7197b7c..3fa59b44d5d 100644 --- a/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb +++ b/db/migrate/20180425205249_add_index_constraints_to_pipeline_iid.rb @@ -6,10 +6,10 @@ class AddIndexConstraintsToPipelineIid < ActiveRecord::Migration disable_ddl_transaction! def up - add_concurrent_index :ci_pipelines, [:project_id, :iid_per_project], unique: true, where: 'iid_per_project IS NOT NULL' + add_concurrent_index :ci_pipelines, [:project_id, :iid], unique: true, where: 'iid IS NOT NULL' end def down - remove_concurrent_index :ci_pipelines, [:project_id, :iid_per_project] + remove_concurrent_index :ci_pipelines, [:project_id, :iid] end end diff --git a/db/schema.rb b/db/schema.rb index af838e109b2..50abe1a8838 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -434,12 +434,12 @@ ActiveRecord::Schema.define(version: 20180425205249) do t.integer "config_source" t.boolean "protected" t.integer "failure_reason" - t.integer "iid_per_project" + t.integer "iid" end add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree - add_index "ci_pipelines", ["project_id", "iid_per_project"], name: "index_ci_pipelines_on_project_id_and_iid_per_project", unique: true, where: "(iid_per_project IS NOT NULL)", using: :btree + add_index "ci_pipelines", ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree -- cgit v1.2.1 From 35cf604b72ec6cabffc46cc7c2da76fb303525cb Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 30 Apr 2018 22:25:56 +0900 Subject: Add to_param override to lookup :id path --- spec/models/ci/pipeline_spec.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index dd94515b0a4..51cdf185c9f 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -35,6 +35,15 @@ describe Ci::Pipeline, :mailer do end end + describe 'modules' do + it_behaves_like 'AtomicInternalId' do + let(:internal_id_attribute) { :iid } + let(:instance) { build(:ci_pipeline) } + let(:scope_attrs) { { project: instance.project } } + let(:usage) { :ci_pipelines } + end + end + describe '#source' do context 'when creating new pipeline' do let(:pipeline) do @@ -173,7 +182,7 @@ describe Ci::Pipeline, :mailer do it 'includes all predefined variables in a valid order' do keys = subject.map { |variable| variable[:key] } - expect(keys).to eq %w[CI_PIPELINE_ID CI_CONFIG_PATH CI_PIPELINE_SOURCE] + expect(keys).to eq %w[CI_PIPELINE_ID CI_PIPELINE_IID CI_CONFIG_PATH CI_PIPELINE_SOURCE] end end -- cgit v1.2.1 From 58f229af992a9481c9eee3165dc5d14eb1dc7d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 3 May 2018 10:48:23 +0200 Subject: Make Atomic Internal ID work for pipelines --- app/models/ci/pipeline.rb | 4 +++- app/models/concerns/atomic_internal_id.rb | 18 +++++++++++------- lib/gitlab/ci/pipeline/chain/create.rb | 3 +++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index efab6f4283a..0622b9c6918 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,9 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.pipelines.count } + has_internal_id :iid, scope: :project, presence: false, to_param: false, init: -> do |s| + s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines.count + end has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 22f516a172f..709589a9128 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -25,9 +25,13 @@ module AtomicInternalId extend ActiveSupport::Concern module ClassMethods - def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName - before_validation(on: :create) do + def has_internal_id(column, scope:, init:, presence: true, to_param: true) # rubocop:disable Naming/PredicateName + before_validation :"ensure_#{column}!", on: :create + validates column, presence: presence, numericality: true + + define_method("ensure_#{column}!") do scope_value = association(scope).reader + if read_attribute(column).blank? && scope_value scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value } usage = self.class.table_name.to_sym @@ -35,13 +39,13 @@ module AtomicInternalId new_iid = InternalId.generate_next(self, scope_attrs, usage, init) write_attribute(column, new_iid) end + + read_attribute(column) end - validates column, presence: true, numericality: true + define_method("to_param") do + read_attribute(column) + end if to_param end end - - def to_param - iid.to_s - end end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index f4c8d5342c1..5967a7a6a58 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,6 +6,9 @@ module Gitlab include Chain::Helpers def perform! + # TODO: allocate next IID outside of transaction + pipeline.ensure_iid! + ::Ci::Pipeline.transaction do pipeline.save! -- cgit v1.2.1 From 07d1d8bd6730015e65bd5123f305bf35b4839237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 3 May 2018 10:50:57 +0200 Subject: Use **CI_PIPELINE_IID** --- app/models/ci/pipeline.rb | 2 +- doc/ci/variables/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 0622b9c6918..84c5dae24bb 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -494,7 +494,7 @@ module Ci def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PIPELINE_ID', value: id.to_s) - .append(key: 'CI_PIPELINE_IID_PER_PROJECT', value: iid.to_s) + .append(key: 'CI_PIPELINE_IID', value: iid.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) end diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 8eb0fc5ea49..2282cc24beb 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -60,7 +60,7 @@ future GitLab releases.** | **CI_RUNNER_REVISION** | all | 10.6 | GitLab Runner revision that is executing the current job | | **CI_RUNNER_EXECUTABLE_ARCH** | all | 10.6 | The OS/architecture of the GitLab Runner executable (note that this is not necessarily the same as the environment of the executor) | | **CI_PIPELINE_ID** | 8.10 | 0.5 | The unique id of the current pipeline that GitLab CI uses internally | -| **CI_PIPELINE_IID_PER_PROJECT** | 10.8 | all | The unique id of the current pipeline that GitLab CI uses internally (Incremented per project) | +| **CI_PIPELINE_IID** | 10.8 | all | The unique id of the current pipeline scoped to project | | **CI_PIPELINE_TRIGGERED** | all | all | The flag to indicate that job was [triggered] | | **CI_PIPELINE_SOURCE** | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, and `pipeline`. For pipelines created before GitLab 9.5, this will show as `unknown` | | **CI_PROJECT_DIR** | all | all | The full path where the repository is cloned and where the job is run | -- cgit v1.2.1 From 0af2ab18fa7914380150c0403289a9d99ea45ded Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 8 May 2018 14:57:41 +0900 Subject: Decouple to_params from AtomicInternalId concern --- app/models/ci/pipeline.rb | 2 +- app/models/concerns/atomic_internal_id.rb | 6 +----- app/models/concerns/iid_routes.rb | 9 +++++++++ app/models/deployment.rb | 1 + app/models/issue.rb | 1 + app/models/merge_request.rb | 1 + app/models/milestone.rb | 1 + 7 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 app/models/concerns/iid_routes.rb diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 6bd2c42bbd3..d542868f01f 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid, scope: :project, presence: false, to_param: false, init: -> do |s| + has_internal_id :iid, scope: :project, presence: false, init: -> do |s| s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines.count end diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 709589a9128..3d867df544f 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -25,7 +25,7 @@ module AtomicInternalId extend ActiveSupport::Concern module ClassMethods - def has_internal_id(column, scope:, init:, presence: true, to_param: true) # rubocop:disable Naming/PredicateName + def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName before_validation :"ensure_#{column}!", on: :create validates column, presence: presence, numericality: true @@ -42,10 +42,6 @@ module AtomicInternalId read_attribute(column) end - - define_method("to_param") do - read_attribute(column) - end if to_param end end end diff --git a/app/models/concerns/iid_routes.rb b/app/models/concerns/iid_routes.rb new file mode 100644 index 00000000000..50957e0230d --- /dev/null +++ b/app/models/concerns/iid_routes.rb @@ -0,0 +1,9 @@ +module IIDRoutes + ## + # This automagically enforces all related routes to use `iid` instead of `id` + # If you want to use `iid` for some routes and `id` for other routes, this module should not to be included, + # instead you should define `iid` or `id` explictly at each route generators. e.g. pipeline_path(project.id, pipeline.iid) + def to_param + iid.to_s + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 254764eefde..dac97676348 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -1,5 +1,6 @@ class Deployment < ActiveRecord::Base include AtomicInternalId + include IIDRoutes belongs_to :project, required: true belongs_to :environment, required: true diff --git a/app/models/issue.rb b/app/models/issue.rb index 0332bfa9371..6d33a67845f 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Issue < ActiveRecord::Base include AtomicInternalId + include IIDRoutes include Issuable include Noteable include Referable diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 628c61d5d69..a14681897fd 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1,5 +1,6 @@ class MergeRequest < ActiveRecord::Base include AtomicInternalId + include IIDRoutes include Issuable include Noteable include Referable diff --git a/app/models/milestone.rb b/app/models/milestone.rb index d14e3a4ded5..f143b8452a2 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -9,6 +9,7 @@ class Milestone < ActiveRecord::Base include CacheMarkdownField include AtomicInternalId + include IIDRoutes include Sortable include Referable include StripAttribute -- cgit v1.2.1 From 04dc80dbb5cb991172ebeb69b9a20c7b6eef4dbf Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 8 May 2018 16:01:18 +0900 Subject: Fix spec --- app/models/ci/pipeline.rb | 2 +- doc/ci/variables/README.md | 2 +- spec/models/ci/pipeline_spec.rb | 1 + .../shared_examples/models/atomic_internal_id_spec.rb | 13 +++++++++++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index d542868f01f..e9a56525fde 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid, scope: :project, presence: false, init: -> do |s| + has_internal_id :iid, scope: :project, presence: false, init: ->(s) do s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines.count end diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 7fa293665e0..4a83d4fbe33 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -63,7 +63,7 @@ future GitLab releases.** | **CI_RUNNER_REVISION** | all | 10.6 | GitLab Runner revision that is executing the current job | | **CI_RUNNER_EXECUTABLE_ARCH** | all | 10.6 | The OS/architecture of the GitLab Runner executable (note that this is not necessarily the same as the environment of the executor) | | **CI_PIPELINE_ID** | 8.10 | 0.5 | The unique id of the current pipeline that GitLab CI uses internally | -| **CI_PIPELINE_IID** | 10.8 | all | The unique id of the current pipeline scoped to project | +| **CI_PIPELINE_IID** | 11.0 | all | The unique id of the current pipeline scoped to project | | **CI_PIPELINE_TRIGGERED** | all | all | The flag to indicate that job was [triggered] | | **CI_PIPELINE_SOURCE** | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, and `pipeline`. For pipelines created before GitLab 9.5, this will show as `unknown` | | **CI_PROJECT_DIR** | all | all | The full path where the repository is cloned and where the job is run | diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 60ba9e62be7..d3e0389cc72 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -41,6 +41,7 @@ describe Ci::Pipeline, :mailer do let(:instance) { build(:ci_pipeline) } let(:scope_attrs) { { project: instance.project } } let(:usage) { :ci_pipelines } + let(:validate_presence) { false } end end diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb index 6a6e13418a9..1bccb578d79 100644 --- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb +++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' -shared_examples_for 'AtomicInternalId' do +shared_examples_for 'AtomicInternalId' do + let(:validate_presence) { true } + describe '.has_internal_id' do describe 'Module inclusion' do subject { described_class } @@ -15,7 +17,14 @@ shared_examples_for 'AtomicInternalId' do allow(InternalId).to receive(:generate_next).and_return(nil) end - it { is_expected.to validate_presence_of(internal_id_attribute) } + it 'checks presence' do + if validate_presence + is_expected.to validate_presence_of(internal_id_attribute) + else + is_expected.not_to validate_presence_of(internal_id_attribute) + end + end + it { is_expected.to validate_numericality_of(internal_id_attribute) } end -- cgit v1.2.1 From 30a6fb64dbce39cc0298e805935bb242c2d7b715 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 10 May 2018 15:56:47 +0900 Subject: Fix atomic internal id spec to allow generate pipeline --- .../shared_examples/models/atomic_internal_id_spec.rb | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb index 1bccb578d79..cd6465675b5 100644 --- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb +++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -shared_examples_for 'AtomicInternalId' do +shared_examples_for 'AtomicInternalId' do let(:validate_presence) { true } describe '.has_internal_id' do @@ -11,21 +11,16 @@ shared_examples_for 'AtomicInternalId' do end describe 'Validation' do - subject { instance } - before do - allow(InternalId).to receive(:generate_next).and_return(nil) + allow_any_instance_of(described_class).to receive(:ensure_iid!) {} end - it 'checks presence' do - if validate_presence - is_expected.to validate_presence_of(internal_id_attribute) - else - is_expected.not_to validate_presence_of(internal_id_attribute) - end - end + it 'validates presence' do + instance.valid? - it { is_expected.to validate_numericality_of(internal_id_attribute) } + expect(instance.errors[:iid]).to include("can't be blank") if validate_presence + expect(instance.errors[:iid]).to include("is not a number") # numericality + end end describe 'Creating an instance' do -- cgit v1.2.1 From 6a108b8fbd5f1b5c2d8e6b51bb34cda06fe0e5ee Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 10 May 2018 16:22:43 +0900 Subject: Fix ensure_iid! method override problem --- app/models/concerns/atomic_internal_id.rb | 4 ++-- lib/gitlab/ci/pipeline/chain/create.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 3d867df544f..876dd0ee1f2 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -26,10 +26,10 @@ module AtomicInternalId module ClassMethods def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName - before_validation :"ensure_#{column}!", on: :create + before_validation :"ensure_#{scope}_#{column}!", on: :create validates column, presence: presence, numericality: true - define_method("ensure_#{column}!") do + define_method("ensure_#{scope}_#{column}!") do scope_value = association(scope).reader if read_attribute(column).blank? && scope_value diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 5967a7a6a58..918a0d151fc 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,8 +6,8 @@ module Gitlab include Chain::Helpers def perform! - # TODO: allocate next IID outside of transaction - pipeline.ensure_iid! + # Allocate next IID outside of transaction + pipeline.ensure_project_iid! ::Ci::Pipeline.transaction do pipeline.save! -- cgit v1.2.1 From 910a7d02a812b1203e320d843a77cad2c7069b63 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 11 May 2018 15:34:36 +0900 Subject: Remove numericality as it's redandant with integer column and validates nil IID --- app/models/concerns/atomic_internal_id.rb | 2 +- spec/lib/gitlab/import_export/safe_model_attributes.yml | 1 + spec/models/ci/build_spec.rb | 1 + spec/support/shared_examples/models/atomic_internal_id_spec.rb | 3 +-- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 876dd0ee1f2..164c704260e 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -27,7 +27,7 @@ module AtomicInternalId module ClassMethods def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName before_validation :"ensure_#{scope}_#{column}!", on: :create - validates column, presence: presence, numericality: true + validates column, presence: presence define_method("ensure_#{scope}_#{column}!") do scope_value = association(scope).reader diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 62da967cf96..2619c6a10d8 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -228,6 +228,7 @@ Ci::Pipeline: - config_source - failure_reason - protected +- iid Ci::Stage: - id - name diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index dc810489011..490b1f0b54e 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1517,6 +1517,7 @@ describe Ci::Build do { key: 'CI_PROJECT_URL', value: project.web_url, public: true }, { key: 'CI_PROJECT_VISIBILITY', value: 'private', public: true }, { key: 'CI_PIPELINE_ID', value: pipeline.id.to_s, public: true }, + { key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s, public: true }, { key: 'CI_CONFIG_PATH', value: pipeline.ci_yaml_file_path, public: true }, { key: 'CI_PIPELINE_SOURCE', value: pipeline.source, public: true }, { key: 'CI_COMMIT_MESSAGE', value: pipeline.git_commit_message, public: true }, diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb index cd6465675b5..ac1cfa47def 100644 --- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb +++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb @@ -12,14 +12,13 @@ shared_examples_for 'AtomicInternalId' do describe 'Validation' do before do - allow_any_instance_of(described_class).to receive(:ensure_iid!) {} + allow_any_instance_of(described_class).to receive(:"ensure_#{scope_attrs.keys.first}_#{internal_id_attribute}!") {} end it 'validates presence' do instance.valid? expect(instance.errors[:iid]).to include("can't be blank") if validate_presence - expect(instance.errors[:iid]).to include("is not a number") # numericality end end -- cgit v1.2.1 From 46fa3089c84642170d86799c4f60fe87507e575d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 11 May 2018 16:49:18 +0900 Subject: Rescue RecordNotUnique when pipeline is created with non-unique iid --- lib/gitlab/ci/pipeline/chain/create.rb | 2 +- spec/lib/gitlab/ci/pipeline/chain/create_spec.rb | 37 ++++++++++++++++------ spec/models/ci/pipeline_spec.rb | 2 +- .../models/atomic_internal_id_spec.rb | 8 +++-- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 918a0d151fc..e62056766bd 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -23,7 +23,7 @@ module Gitlab end end end - rescue ActiveRecord::RecordInvalid => e + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique => e error("Failed to persist the pipeline: #{e}") end diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index 0edc3f315bb..b8ab0135092 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -37,21 +37,38 @@ describe Gitlab::Ci::Pipeline::Chain::Create do end context 'when pipeline has validation errors' do - let(:pipeline) do - build(:ci_pipeline, project: project, ref: nil) + shared_examples_for 'expectations' do + it 'breaks the chain' do + expect(step.break?).to be true + end + + it 'appends validation error' do + expect(pipeline.errors.to_a) + .to include /Failed to persist the pipeline/ + end end + + context 'when ref is nil' do + let(:pipeline) do + build(:ci_pipeline, project: project, ref: nil) + end - before do - step.perform! - end + before do + step.perform! + end - it 'breaks the chain' do - expect(step.break?).to be true + it_behaves_like 'expectations' end - it 'appends validation error' do - expect(pipeline.errors.to_a) - .to include /Failed to persist the pipeline/ + context 'when pipeline has a duplicate iid' do + before do + allow_any_instance_of(Ci::Pipeline).to receive(:ensure_project_iid!) { |p| p.send(:write_attribute, :iid, 1) } + create(:ci_pipeline, project: project) + + step.perform! + end + + it_behaves_like 'expectations' end end end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index d3e0389cc72..c10d0eb55da 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -41,7 +41,7 @@ describe Ci::Pipeline, :mailer do let(:instance) { build(:ci_pipeline) } let(:scope_attrs) { { project: instance.project } } let(:usage) { :ci_pipelines } - let(:validate_presence) { false } + let(:allow_nil) { true } end end diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb index ac1cfa47def..dce2622172b 100644 --- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb +++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' shared_examples_for 'AtomicInternalId' do - let(:validate_presence) { true } + let(:allow_nil) { false } describe '.has_internal_id' do describe 'Module inclusion' do @@ -18,7 +18,11 @@ shared_examples_for 'AtomicInternalId' do it 'validates presence' do instance.valid? - expect(instance.errors[:iid]).to include("can't be blank") if validate_presence + if allow_nil + expect(instance.errors[:iid]).to be_empty + else + expect(instance.errors[:iid]).to include("can't be blank") + end end end -- cgit v1.2.1 From a74184eb5e692ef77fe3be28b1f4a40549c8fcff Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 11 May 2018 16:52:48 +0900 Subject: Fix static analysys --- app/models/ci/pipeline.rb | 2 +- app/models/concerns/iid_routes.rb | 2 +- app/models/deployment.rb | 2 +- app/models/issue.rb | 2 +- app/models/merge_request.rb | 2 +- app/models/milestone.rb | 2 +- spec/lib/gitlab/ci/pipeline/chain/create_spec.rb | 6 +++--- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index e9a56525fde..accd0f11a9b 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -15,7 +15,7 @@ module Ci belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' has_internal_id :iid, scope: :project, presence: false, init: ->(s) do - s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines.count + s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines&.count end has_many :stages diff --git a/app/models/concerns/iid_routes.rb b/app/models/concerns/iid_routes.rb index 50957e0230d..246748cf52c 100644 --- a/app/models/concerns/iid_routes.rb +++ b/app/models/concerns/iid_routes.rb @@ -1,4 +1,4 @@ -module IIDRoutes +module IidRoutes ## # This automagically enforces all related routes to use `iid` instead of `id` # If you want to use `iid` for some routes and `id` for other routes, this module should not to be included, diff --git a/app/models/deployment.rb b/app/models/deployment.rb index dac97676348..ac86e9e8de0 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -1,6 +1,6 @@ class Deployment < ActiveRecord::Base include AtomicInternalId - include IIDRoutes + include IidRoutes belongs_to :project, required: true belongs_to :environment, required: true diff --git a/app/models/issue.rb b/app/models/issue.rb index 6d33a67845f..d0e8fe908e4 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -2,7 +2,7 @@ require 'carrierwave/orm/activerecord' class Issue < ActiveRecord::Base include AtomicInternalId - include IIDRoutes + include IidRoutes include Issuable include Noteable include Referable diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a14681897fd..f42d432cc66 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1,6 +1,6 @@ class MergeRequest < ActiveRecord::Base include AtomicInternalId - include IIDRoutes + include IidRoutes include Issuable include Noteable include Referable diff --git a/app/models/milestone.rb b/app/models/milestone.rb index f143b8452a2..d05dcfd083a 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -9,7 +9,7 @@ class Milestone < ActiveRecord::Base include CacheMarkdownField include AtomicInternalId - include IIDRoutes + include IidRoutes include Sortable include Referable include StripAttribute diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index b8ab0135092..9c0bedfb6b7 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -41,13 +41,13 @@ describe Gitlab::Ci::Pipeline::Chain::Create do it 'breaks the chain' do expect(step.break?).to be true end - + it 'appends validation error' do expect(pipeline.errors.to_a) .to include /Failed to persist the pipeline/ end end - + context 'when ref is nil' do let(:pipeline) do build(:ci_pipeline, project: project, ref: nil) @@ -64,7 +64,7 @@ describe Gitlab::Ci::Pipeline::Chain::Create do before do allow_any_instance_of(Ci::Pipeline).to receive(:ensure_project_iid!) { |p| p.send(:write_attribute, :iid, 1) } create(:ci_pipeline, project: project) - + step.perform! end -- cgit v1.2.1 From 82a49d0fea1ace87b5619fbc4ed728f43fdef7bd Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 14 May 2018 17:41:56 +0900 Subject: Clarify scope for AtomicInternalId shared spec --- spec/models/ci/pipeline_spec.rb | 1 + spec/models/deployment_spec.rb | 1 + spec/models/issue_spec.rb | 1 + spec/models/merge_request_spec.rb | 1 + spec/models/milestone_spec.rb | 2 ++ spec/support/shared_examples/models/atomic_internal_id_spec.rb | 6 +++--- 6 files changed, 9 insertions(+), 3 deletions(-) diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index c10d0eb55da..e0fe62b39e6 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -39,6 +39,7 @@ describe Ci::Pipeline, :mailer do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:ci_pipeline) } + let(:scope) { :project } let(:scope_attrs) { { project: instance.project } } let(:usage) { :ci_pipelines } let(:allow_nil) { true } diff --git a/spec/models/deployment_spec.rb b/spec/models/deployment_spec.rb index aee70bcfb29..e01906f4b6c 100644 --- a/spec/models/deployment_spec.rb +++ b/spec/models/deployment_spec.rb @@ -20,6 +20,7 @@ describe Deployment do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:deployment) } + let(:scope) { :project } let(:scope_attrs) { { project: instance.project } } let(:usage) { :deployments } end diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index 128acf83686..e818fbeb9cf 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -17,6 +17,7 @@ describe Issue do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:issue) } + let(:scope) { :project } let(:scope_attrs) { { project: instance.project } } let(:usage) { :issues } end diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 04379e7d2c3..c2ab1259ebf 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -25,6 +25,7 @@ describe MergeRequest do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:merge_request) } + let(:scope) { :target_project } let(:scope_attrs) { { project: instance.target_project } } let(:usage) { :merge_requests } end diff --git a/spec/models/milestone_spec.rb b/spec/models/milestone_spec.rb index 4bb9717d33e..204d6b47832 100644 --- a/spec/models/milestone_spec.rb +++ b/spec/models/milestone_spec.rb @@ -6,6 +6,7 @@ describe Milestone do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:milestone, project: build(:project), group: nil) } + let(:scope) { :project } let(:scope_attrs) { { project: instance.project } } let(:usage) { :milestones } end @@ -15,6 +16,7 @@ describe Milestone do it_behaves_like 'AtomicInternalId' do let(:internal_id_attribute) { :iid } let(:instance) { build(:milestone, project: nil, group: build(:group)) } + let(:scope) { :group } let(:scope_attrs) { { namespace: instance.group } } let(:usage) { :milestones } end diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb index dce2622172b..a05279364f2 100644 --- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb +++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb @@ -12,16 +12,16 @@ shared_examples_for 'AtomicInternalId' do describe 'Validation' do before do - allow_any_instance_of(described_class).to receive(:"ensure_#{scope_attrs.keys.first}_#{internal_id_attribute}!") {} + allow_any_instance_of(described_class).to receive(:"ensure_#{scope}_#{internal_id_attribute}!") {} end it 'validates presence' do instance.valid? if allow_nil - expect(instance.errors[:iid]).to be_empty + expect(instance.errors[internal_id_attribute]).to be_empty else - expect(instance.errors[:iid]).to include("can't be blank") + expect(instance.errors[internal_id_attribute]).to include("can't be blank") end end end -- cgit v1.2.1 From 8e92e25b62ca108de775362e6d2981e54535f094 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 22 May 2018 15:21:45 +0900 Subject: Remvoe disable_ddl_transaction! and redandant RecordNotUnique exception rescue --- ...80424160449_add_pipeline_iid_to_ci_pipelines.rb | 2 -- lib/gitlab/ci/pipeline/chain/create.rb | 2 +- spec/lib/gitlab/ci/pipeline/chain/create_spec.rb | 27 +++++----------------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb index d732116e9ce..e8f0c91d612 100644 --- a/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb +++ b/db/migrate/20180424160449_add_pipeline_iid_to_ci_pipelines.rb @@ -3,8 +3,6 @@ class AddPipelineIidToCiPipelines < ActiveRecord::Migration DOWNTIME = false - disable_ddl_transaction! - def up add_column :ci_pipelines, :iid, :integer end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index e62056766bd..918a0d151fc 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -23,7 +23,7 @@ module Gitlab end end end - rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique => e + rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") end diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index 9c0bedfb6b7..de69a65aee4 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -37,17 +37,6 @@ describe Gitlab::Ci::Pipeline::Chain::Create do end context 'when pipeline has validation errors' do - shared_examples_for 'expectations' do - it 'breaks the chain' do - expect(step.break?).to be true - end - - it 'appends validation error' do - expect(pipeline.errors.to_a) - .to include /Failed to persist the pipeline/ - end - end - context 'when ref is nil' do let(:pipeline) do build(:ci_pipeline, project: project, ref: nil) @@ -57,18 +46,14 @@ describe Gitlab::Ci::Pipeline::Chain::Create do step.perform! end - it_behaves_like 'expectations' - end - - context 'when pipeline has a duplicate iid' do - before do - allow_any_instance_of(Ci::Pipeline).to receive(:ensure_project_iid!) { |p| p.send(:write_attribute, :iid, 1) } - create(:ci_pipeline, project: project) - - step.perform! + it 'breaks the chain' do + expect(step.break?).to be true end - it_behaves_like 'expectations' + it 'appends validation error' do + expect(pipeline.errors.to_a) + .to include /Failed to persist the pipeline/ + end end end end -- cgit v1.2.1 From aac8d1f3363a87f0bcd31009aad41d577f0a3f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 28 May 2018 21:10:51 +0200 Subject: Add check for nil auto_devops in Projects::UpdateService --- app/services/projects/update_service.rb | 2 +- spec/services/projects/update_service_spec.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index 679f4a9cb62..1cea110d555 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -36,7 +36,7 @@ module Projects end def run_auto_devops_pipeline? - return false if project.repository.gitlab_ci_yml || !project.auto_devops.previous_changes.include?('enabled') + return false if project.repository.gitlab_ci_yml || project.auto_devops.nil? || !project.auto_devops.previous_changes.include?('enabled') project.auto_devops.enabled? || (project.auto_devops.enabled.nil? && Gitlab::CurrentSettings.auto_devops_enabled?) end diff --git a/spec/services/projects/update_service_spec.rb b/spec/services/projects/update_service_spec.rb index 3e6073b9861..1f761bcbbad 100644 --- a/spec/services/projects/update_service_spec.rb +++ b/spec/services/projects/update_service_spec.rb @@ -275,6 +275,10 @@ describe Projects::UpdateService do it { is_expected.to eq(false) } end + context 'when auto devops is nil' do + it { is_expected.to eq(false) } + end + context 'when auto devops is explicitly enabled' do before do project.create_auto_devops!(enabled: true) -- cgit v1.2.1 From ca1d6d2f6dfd203e8b25e19ca1709ba722d922d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 28 May 2018 21:14:19 +0200 Subject: Add CHANGELOG entry --- ...ethoderror-undefined-method-previous_changes-for-nil-nilclass.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/46452-nomethoderror-undefined-method-previous_changes-for-nil-nilclass.yml diff --git a/changelogs/unreleased/46452-nomethoderror-undefined-method-previous_changes-for-nil-nilclass.yml b/changelogs/unreleased/46452-nomethoderror-undefined-method-previous_changes-for-nil-nilclass.yml new file mode 100644 index 00000000000..89dee65f5a8 --- /dev/null +++ b/changelogs/unreleased/46452-nomethoderror-undefined-method-previous_changes-for-nil-nilclass.yml @@ -0,0 +1,5 @@ +--- +title: Check for nil AutoDevOps when saving project CI/CD settings. +merge_request: 19190 +author: +type: fixed -- cgit v1.2.1 From b5c706326ada2c0d213dd512842c5f677d9d94f9 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 19 May 2018 06:03:29 -0700 Subject: Upgrade to Ruby 2.4.4 Fixes that make this work: * A change in Ruby (https://github.com/ruby/ruby/commit/ce635262f53b760284d56bb1027baebaaec175d1) requires passing in the exact required length for OpenSSL keys and IVs. * Ensure the secrets.yml is generated before any prepended modules are loaded. This is done by renaming the `secret_token.rb` initializer to `01_secret_token.rb`, which is a bit ugly but involves the least impact on other files. --- .gitlab-ci.yml | 6 +- .ruby-version | 2 +- app/models/clusters/platforms/kubernetes.rb | 4 +- app/models/clusters/providers/gcp.rb | 2 +- app/models/concerns/has_variable.rb | 2 +- app/models/pages_domain.rb | 2 +- app/models/project_import_data.rb | 2 +- app/models/remote_mirror.rb | 2 +- config/initializers/01_secret_token.rb | 95 ++++++++++++++++++++++ config/initializers/secret_token.rb | 92 --------------------- config/settings.rb | 4 + ...152808_remove_wrong_import_url_from_projects.rb | 2 +- ...rnetes_service_to_new_clusters_architectures.rb | 2 +- doc/install/installation.md | 6 +- spec/initializers/secret_token_spec.rb | 2 +- spec/models/concerns/has_variable_spec.rb | 4 +- 16 files changed, 119 insertions(+), 110 deletions(-) create mode 100644 config/initializers/01_secret_token.rb delete mode 100644 config/initializers/secret_token.rb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b2ee4b1cd8..7f3548feac3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.7-golang-1.9-git-2.17-chrome-65.0-node-8.x-yarn-1.2-postgresql-9.6" +image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.4.4-golang-1.9-git-2.17-chrome-65.0-node-8.x-yarn-1.2-postgresql-9.6" .dedicated-runner: &dedicated-runner retry: 1 @@ -6,7 +6,7 @@ image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.7-golang-1.9-git - gitlab-org .default-cache: &default-cache - key: "ruby-2.3.7-debian-stretch-with-yarn" + key: "ruby-2.4.4-debian-stretch-with-yarn" paths: - vendor/ruby - .yarn-cache/ @@ -550,7 +550,7 @@ static-analysis: script: - scripts/static-analysis cache: - key: "ruby-2.3.7-debian-stretch-with-yarn-and-rubocop" + key: "ruby-2.4.4-debian-stretch-with-yarn-and-rubocop" paths: - vendor/ruby - .yarn-cache/ diff --git a/.ruby-version b/.ruby-version index 00355e29d11..79a614418f7 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.3.7 +2.4.4 diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index ba6552f238f..25eac5160f1 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -11,12 +11,12 @@ module Clusters attr_encrypted :password, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' attr_encrypted :token, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' before_validation :enforce_namespace_to_lower_case diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index 7fac32466ab..eb2e42fd3fe 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -11,7 +11,7 @@ module Clusters attr_encrypted :access_token, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' validates :gcp_project_id, diff --git a/app/models/concerns/has_variable.rb b/app/models/concerns/has_variable.rb index 8a241e4374a..c8e20c0ab81 100644 --- a/app/models/concerns/has_variable.rb +++ b/app/models/concerns/has_variable.rb @@ -13,7 +13,7 @@ module HasVariable attr_encrypted :value, mode: :per_attribute_iv_and_salt, insecure_mode: true, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' def key=(new_key) diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 2e478a24778..bfea64c3759 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -19,7 +19,7 @@ class PagesDomain < ActiveRecord::Base attr_encrypted :key, mode: :per_attribute_iv_and_salt, insecure_mode: true, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' after_initialize :set_verification_code diff --git a/app/models/project_import_data.rb b/app/models/project_import_data.rb index 6da6632f4f2..1d7089ccfc7 100644 --- a/app/models/project_import_data.rb +++ b/app/models/project_import_data.rb @@ -3,7 +3,7 @@ require 'carrierwave/orm/activerecord' class ProjectImportData < ActiveRecord::Base belongs_to :project, inverse_of: :import_data attr_encrypted :credentials, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, marshal: true, encode: true, mode: :per_attribute_iv_and_salt, diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index bbf8fd9c6a7..aba1f2f384f 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -5,7 +5,7 @@ class RemoteMirror < ActiveRecord::Base UNPROTECTED_BACKOFF_DELAY = 5.minutes attr_encrypted :credentials, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, marshal: true, encode: true, mode: :per_attribute_iv_and_salt, diff --git a/config/initializers/01_secret_token.rb b/config/initializers/01_secret_token.rb new file mode 100644 index 00000000000..02bded43083 --- /dev/null +++ b/config/initializers/01_secret_token.rb @@ -0,0 +1,95 @@ +# This file needs to be loaded BEFORE any initializers that attempt to +# prepend modules that require access to secrets (e.g. EE's 0_as_concern.rb). +# +# Be sure to restart your server when you modify this file. + +require 'securerandom' + +# Transition material in .secret to the secret_key_base key in config/secrets.yml. +# Historically, ENV['SECRET_KEY_BASE'] takes precedence over .secret, so we maintain that +# behavior. +# +# It also used to be the case that the key material in ENV['SECRET_KEY_BASE'] or .secret +# was used to encrypt OTP (two-factor authentication) data so if present, we copy that key +# material into config/secrets.yml under otp_key_base. +# +# Finally, if we have successfully migrated all secrets to config/secrets.yml, delete the +# .secret file to avoid confusion. +# +def create_tokens + secret_file = Rails.root.join('.secret') + file_secret_key = File.read(secret_file).chomp if File.exist?(secret_file) + env_secret_key = ENV['SECRET_KEY_BASE'] + + # Ensure environment variable always overrides secrets.yml. + Rails.application.secrets.secret_key_base = env_secret_key if env_secret_key.present? + + defaults = { + secret_key_base: file_secret_key || generate_new_secure_token, + otp_key_base: env_secret_key || file_secret_key || generate_new_secure_token, + db_key_base: generate_new_secure_token, + openid_connect_signing_key: generate_new_rsa_private_key + } + + missing_secrets = set_missing_keys(defaults) + write_secrets_yml(missing_secrets) unless missing_secrets.empty? + + begin + File.delete(secret_file) if file_secret_key + rescue => e + warn "Error deleting useless .secret file: #{e}" + end +end + +def generate_new_secure_token + SecureRandom.hex(64) +end + +def generate_new_rsa_private_key + OpenSSL::PKey::RSA.new(2048).to_pem +end + +def warn_missing_secret(secret) + warn "Missing Rails.application.secrets.#{secret} for #{Rails.env} environment. The secret will be generated and stored in config/secrets.yml." +end + +def set_missing_keys(defaults) + defaults.stringify_keys.each_with_object({}) do |(key, default), missing| + if Rails.application.secrets[key].blank? + warn_missing_secret(key) + + missing[key] = Rails.application.secrets[key] = default + end + end +end + +def write_secrets_yml(missing_secrets) + secrets_yml = Rails.root.join('config/secrets.yml') + rails_env = Rails.env.to_s + secrets = YAML.load_file(secrets_yml) if File.exist?(secrets_yml) + secrets ||= {} + secrets[rails_env] ||= {} + + secrets[rails_env].merge!(missing_secrets) do |key, old, new| + # Previously, it was possible this was set to the literal contents of an Erb + # expression that evaluated to an empty value. We don't want to support that + # specifically, just ensure we don't break things further. + # + if old.present? + warn < e - warn "Error deleting useless .secret file: #{e}" - end -end - -def generate_new_secure_token - SecureRandom.hex(64) -end - -def generate_new_rsa_private_key - OpenSSL::PKey::RSA.new(2048).to_pem -end - -def warn_missing_secret(secret) - warn "Missing Rails.application.secrets.#{secret} for #{Rails.env} environment. The secret will be generated and stored in config/secrets.yml." -end - -def set_missing_keys(defaults) - defaults.stringify_keys.each_with_object({}) do |(key, default), missing| - if Rails.application.secrets[key].blank? - warn_missing_secret(key) - - missing[key] = Rails.application.secrets[key] = default - end - end -end - -def write_secrets_yml(missing_secrets) - secrets_yml = Rails.root.join('config/secrets.yml') - rails_env = Rails.env.to_s - secrets = YAML.load_file(secrets_yml) if File.exist?(secrets_yml) - secrets ||= {} - secrets[rails_env] ||= {} - - secrets[rails_env].merge!(missing_secrets) do |key, old, new| - # Previously, it was possible this was set to the literal contents of an Erb - # expression that evaluated to an empty value. We don't want to support that - # specifically, just ensure we don't break things further. - # - if old.present? - warn < :per_attribute_iv_and_salt, diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb index 11b581e4b57..1586a7eb92f 100644 --- a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb +++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb @@ -48,7 +48,7 @@ class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migrati attr_encrypted :token, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' end diff --git a/doc/install/installation.md b/doc/install/installation.md index a0ae9017f71..34268c67140 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -133,9 +133,9 @@ Remove the old Ruby 1.8 if present: Download Ruby and compile it: mkdir /tmp/ruby && cd /tmp/ruby - curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.7.tar.gz - echo '540996fec64984ab6099e34d2f5820b14904f15a ruby-2.3.7.tar.gz' | shasum -c - && tar xzf ruby-2.3.7.tar.gz - cd ruby-2.3.7 + curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.4/ruby-2.4.4.tar.gz + echo 'ec82b0d53bd0adad9b19e6b45e44d54e9ec3f10c ruby-2.4.4.tar.gz' | shasum -c - && tar xzf ruby-2.4.4.tar.gz + cd ruby-2.4.4 ./configure --disable-install-rdoc make diff --git a/spec/initializers/secret_token_spec.rb b/spec/initializers/secret_token_spec.rb index d56e14e0e0b..c3dfd7bedbe 100644 --- a/spec/initializers/secret_token_spec.rb +++ b/spec/initializers/secret_token_spec.rb @@ -1,5 +1,5 @@ require 'spec_helper' -require_relative '../../config/initializers/secret_token' +require_relative '../../config/initializers/01_secret_token' describe 'create_tokens' do include StubENV diff --git a/spec/models/concerns/has_variable_spec.rb b/spec/models/concerns/has_variable_spec.rb index f87869a2fdc..3fbe86c5b56 100644 --- a/spec/models/concerns/has_variable_spec.rb +++ b/spec/models/concerns/has_variable_spec.rb @@ -45,8 +45,10 @@ describe HasVariable do end it 'fails to decrypt if iv is incorrect' do - subject.encrypted_value_iv = SecureRandom.hex + # attr_encrypted expects the IV to be 16 bytes and base64-encoded + subject.encrypted_value_iv = [SecureRandom.hex(8)].pack('m') subject.instance_variable_set(:@value, nil) + expect { subject.value } .to raise_error(OpenSSL::Cipher::CipherError, 'bad decrypt') end -- cgit v1.2.1 From 59e1e9710898c4547c79a3d5eef39a3f88d3bd7a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 30 May 2018 15:17:09 +0900 Subject: Add build_relations method in Chain::Populate --- lib/gitlab/ci/pipeline/chain/create.rb | 3 --- lib/gitlab/ci/pipeline/chain/populate.rb | 17 +++++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 918a0d151fc..f4c8d5342c1 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,9 +6,6 @@ module Gitlab include Chain::Helpers def perform! - # Allocate next IID outside of transaction - pipeline.ensure_project_iid! - ::Ci::Pipeline.transaction do pipeline.save! diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 69b8a8fc68f..7a2a1c6a80b 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -8,10 +8,7 @@ module Gitlab PopulateError = Class.new(StandardError) def perform! - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) + build_relations ## # Populate pipeline with all stages, and stages with builds. @@ -34,6 +31,18 @@ module Gitlab def break? pipeline.errors.any? end + + private + + def build_relations + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + # Allocate next IID. This operation must be outside of transactions of pipeline creations. + pipeline.ensure_project_iid! + end end end end -- cgit v1.2.1 From 0e22b50df8b269ccae32ab68b9ba26e7eea861b0 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 30 May 2018 16:42:55 +0900 Subject: Add spec for variables expression --- spec/lib/gitlab/ci/pipeline/chain/create_spec.rb | 32 ++++++++++------------ spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb | 24 ++++++++++++++++ spec/models/ci/pipeline_spec.rb | 14 ++++++++++ 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index de69a65aee4..0edc3f315bb 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -37,23 +37,21 @@ describe Gitlab::Ci::Pipeline::Chain::Create do end context 'when pipeline has validation errors' do - context 'when ref is nil' do - let(:pipeline) do - build(:ci_pipeline, project: project, ref: nil) - end - - before do - step.perform! - end - - it 'breaks the chain' do - expect(step.break?).to be true - end - - it 'appends validation error' do - expect(pipeline.errors.to_a) - .to include /Failed to persist the pipeline/ - end + let(:pipeline) do + build(:ci_pipeline, project: project, ref: nil) + end + + before do + step.perform! + end + + it 'breaks the chain' do + expect(step.break?).to be true + end + + it 'appends validation error' do + expect(pipeline.errors.to_a) + .to include /Failed to persist the pipeline/ end end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index 4d7d6951a51..bcfa9f0c282 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -42,6 +42,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do it 'correctly assigns user' do expect(pipeline.builds).to all(have_attributes(user: user)) end + + it 'has pipeline iid' do + expect(pipeline.iid).to be > 0 + end end context 'when pipeline is empty' do @@ -68,6 +72,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do expect(pipeline.errors.to_a) .to include 'No stages / jobs for this pipeline.' end + + it 'wastes pipeline iid' do + expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0 + end end context 'when pipeline has validation errors' do @@ -87,6 +95,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do expect(pipeline.errors.to_a) .to include 'Failed to build the pipeline!' end + + it 'wastes pipeline iid' do + expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0 + end end context 'when there is a seed blocks present' do @@ -111,6 +123,12 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do expect(pipeline.variables.first.key).to eq 'VAR' expect(pipeline.variables.first.value).to eq '123' end + + it 'has pipeline iid' do + step.perform! + + expect(pipeline.iid).to be > 0 + end end context 'when seeds block tries to persist some resources' do @@ -121,6 +139,12 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do it 'raises exception' do expect { step.perform! }.to raise_error(ActiveRecord::RecordNotSaved) end + + it 'does not waste pipeline iid' do + step.perform rescue nil + + expect(InternalId.ci_pipelines.where(project_id: project.id).exists?).to be_falsy + end end end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 314cb3a28ed..7d28f2eb86b 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -397,6 +397,20 @@ describe Ci::Pipeline, :mailer do expect(seeds.size).to eq 1 expect(seeds.dig(0, 0, :name)).to eq 'unit' end + + context "when pipeline iid is used for 'only' keyword" do + let(:config) do + { rspec: { script: 'rspec', only: { variables: ['$CI_PIPELINE_IID == 2'] } }, + prod: { script: 'cap prod', only: { variables: ['$CI_PIPELINE_IID == 1'] } } } + end + + it 'returns stage seeds only when variables expression is truthy' do + seeds = pipeline.stage_seeds + + expect(seeds.size).to eq 1 + expect(seeds.dig(0, 0, :name)).to eq 'prod' + end + end end end -- cgit v1.2.1 From d10a70a0c56e648d961f4246ab4f64dcee0da5ac Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 30 May 2018 09:47:38 -0700 Subject: Fix missing squash parameter in doc/api/merge_requests.md --- doc/api/merge_requests.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 8849f490c4f..8dbe35d4ac6 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -476,6 +476,7 @@ Parameters: "changes_count": "1", "should_remove_source_branch": true, "force_remove_source_branch": false, + "squash": false, "web_url": "http://example.com/example/example/merge_requests/1", "discussion_locked": false, "time_stats": { -- cgit v1.2.1 From b6caf937ccc780bd2abd7c4518f85ebcad12548f Mon Sep 17 00:00:00 2001 From: Maxime Guyot Date: Thu, 31 May 2018 22:10:56 +0200 Subject: Update Doorkeeper OpenID for Kubernetes support --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4d2bd62bec0..9fc895e42be 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -172,7 +172,7 @@ GEM unf (>= 0.0.5, < 1.0.0) doorkeeper (4.3.2) railties (>= 4.2) - doorkeeper-openid_connect (1.3.0) + doorkeeper-openid_connect (1.4.0) doorkeeper (~> 4.3) json-jwt (~> 1.6) dropzonejs-rails (0.7.2) -- cgit v1.2.1 From 0d44f4d50ef175997fe1f90de9e622a4f3b867e3 Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Wed, 23 May 2018 09:54:57 +0800 Subject: Rephrase "maintainer" to more precise "members who can merge to the target branch" "Maintainer" will be freed to be used for #42751 --- .../vue_merge_request_widget/mr_widget_options.vue | 4 +- .../stores/mr_widget_store.js | 2 +- .../merge_requests/application_controller.rb | 2 +- app/helpers/merge_requests_helper.rb | 4 +- app/models/merge_request.rb | 12 +-- app/models/project.rb | 14 ++-- app/policies/ci/build_policy.rb | 6 +- app/policies/ci/pipeline_policy.rb | 6 +- app/serializers/merge_request_widget_entity.rb | 2 +- app/services/merge_requests/base_service.rb | 4 +- .../shared/issuable/form/_contribution.html.haml | 10 +-- .../shared/projects/_edit_information.html.haml | 2 +- .../unreleased/42751-rename-mr-maintainer-push.yml | 5 ++ ...name_merge_requests_allow_maintainer_to_push.rb | 15 ++++ ...rge_requests_allow_maintainer_to_push_rename.rb | 15 ++++ db/schema.rb | 2 +- doc/api/merge_requests.md | 8 +- .../project/merge_requests/allow_collaboration.md | 20 +++++ .../merge_requests/img/allow_collaboration.png | Bin 0 -> 39513 bytes .../merge_requests/img/allow_maintainer_push.png | Bin 49216 -> 0 bytes doc/user/project/merge_requests/index.md | 2 +- .../project/merge_requests/maintainer_access.md | 21 +---- lib/api/entities.rb | 4 +- lib/api/merge_requests.rb | 3 +- lib/gitlab/user_access.rb | 2 +- locale/gitlab.pot | 4 +- .../merge_request/maintainer_edits_fork_spec.rb | 2 +- .../user_allows_a_maintainer_to_push_spec.rb | 85 --------------------- ...ows_commits_from_memebers_who_can_merge_spec.rb | 85 +++++++++++++++++++++ .../api/schemas/entities/merge_request_basic.json | 1 + .../api/schemas/entities/merge_request_widget.json | 2 +- .../api/schemas/public_api/v4/merge_requests.json | 1 + .../gitlab/import_export/safe_model_attributes.yml | 2 +- spec/lib/gitlab/user_access_spec.rb | 2 +- spec/models/merge_request_spec.rb | 28 +++---- spec/models/project_spec.rb | 22 +++--- spec/policies/ci/build_policy_spec.rb | 2 +- spec/policies/ci/pipeline_policy_spec.rb | 2 +- spec/policies/project_policy_spec.rb | 2 +- spec/requests/api/merge_requests_spec.rb | 12 +-- spec/services/ci/retry_pipeline_service_spec.rb | 4 +- .../services/merge_requests/update_service_spec.rb | 12 +-- 42 files changed, 240 insertions(+), 193 deletions(-) create mode 100644 changelogs/unreleased/42751-rename-mr-maintainer-push.yml create mode 100644 db/migrate/20180523042841_rename_merge_requests_allow_maintainer_to_push.rb create mode 100644 db/post_migrate/20180523125103_cleanup_merge_requests_allow_maintainer_to_push_rename.rb create mode 100644 doc/user/project/merge_requests/allow_collaboration.md create mode 100644 doc/user/project/merge_requests/img/allow_collaboration.png delete mode 100644 doc/user/project/merge_requests/img/allow_maintainer_push.png delete mode 100644 spec/features/merge_request/user_allows_a_maintainer_to_push_spec.rb create mode 100644 spec/features/merge_request/user_allows_commits_from_memebers_who_can_merge_spec.rb diff --git a/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.vue b/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.vue index f69fe03fcb3..c20d07a169d 100644 --- a/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.vue +++ b/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.vue @@ -265,10 +265,10 @@ export default { />
- {{ s__("mrWidget|Allows edits from maintainers") }} + {{ s__("mrWidget|Allows commits from members who can merge to the target branch") }}
Gitlab::VisibilityLevel::PRIVATE && source_project.visibility_level > Gitlab::VisibilityLevel::PRIVATE && !ProtectedBranch.protected?(source_project, source_branch) end - def can_allow_maintainer_to_push?(user) - maintainer_push_possible? && + def can_allow_collaboration?(user) + collaborative_push_possible? && Ability.allowed?(user, :push_code, source_project) end diff --git a/app/models/project.rb b/app/models/project.rb index e275ac4dc6f..73b808b4cb5 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1973,18 +1973,18 @@ class Project < ActiveRecord::Base .limit(1) .select(1) source_of_merge_requests.opened - .where(allow_maintainer_to_push: true) + .where(allow_collaboration: true) .where('EXISTS (?)', developer_access_exists) end - def branch_allows_maintainer_push?(user, branch_name) + def branch_allows_collaboration?(user, branch_name) return false unless user cache_key = "user:#{user.id}:#{branch_name}:branch_allows_push" - memoized_results = strong_memoize(:branch_allows_maintainer_push) do + memoized_results = strong_memoize(:branch_allows_collaboration) do Hash.new do |result, cache_key| - result[cache_key] = fetch_branch_allows_maintainer_push?(user, branch_name) + result[cache_key] = fetch_branch_allows_collaboration?(user, branch_name) end end @@ -2126,18 +2126,18 @@ class Project < ActiveRecord::Base raise ex end - def fetch_branch_allows_maintainer_push?(user, branch_name) + def fetch_branch_allows_collaboration?(user, branch_name) check_access = -> do next false if empty_repo? merge_request = source_of_merge_requests.opened - .where(allow_maintainer_to_push: true) + .where(allow_collaboration: true) .find_by(source_branch: branch_name) merge_request&.can_be_merged_by?(user) end if RequestStore.active? - RequestStore.fetch("project-#{id}:branch-#{branch_name}:user-#{user.id}:branch_allows_maintainer_push") do + RequestStore.fetch("project-#{id}:branch-#{branch_name}:user-#{user.id}:branch_allows_collaboration") do check_access.call end else diff --git a/app/policies/ci/build_policy.rb b/app/policies/ci/build_policy.rb index 8b65758f3e8..1c0cc7425ec 100644 --- a/app/policies/ci/build_policy.rb +++ b/app/policies/ci/build_policy.rb @@ -14,8 +14,8 @@ module Ci @subject.triggered_by?(@user) end - condition(:branch_allows_maintainer_push) do - @subject.project.branch_allows_maintainer_push?(@user, @subject.ref) + condition(:branch_allows_collaboration) do + @subject.project.branch_allows_collaboration?(@user, @subject.ref) end rule { protected_ref }.policy do @@ -25,7 +25,7 @@ module Ci rule { can?(:admin_build) | (can?(:update_build) & owner_of_job) }.enable :erase_build - rule { can?(:public_access) & branch_allows_maintainer_push }.policy do + rule { can?(:public_access) & branch_allows_collaboration }.policy do enable :update_build enable :update_commit_status end diff --git a/app/policies/ci/pipeline_policy.rb b/app/policies/ci/pipeline_policy.rb index 540e4235299..b81329d0625 100644 --- a/app/policies/ci/pipeline_policy.rb +++ b/app/policies/ci/pipeline_policy.rb @@ -4,13 +4,13 @@ module Ci condition(:protected_ref) { ref_protected?(@user, @subject.project, @subject.tag?, @subject.ref) } - condition(:branch_allows_maintainer_push) do - @subject.project.branch_allows_maintainer_push?(@user, @subject.ref) + condition(:branch_allows_collaboration) do + @subject.project.branch_allows_collaboration?(@user, @subject.ref) end rule { protected_ref }.prevent :update_pipeline - rule { can?(:public_access) & branch_allows_maintainer_push }.policy do + rule { can?(:public_access) & branch_allows_collaboration }.policy do enable :update_pipeline end diff --git a/app/serializers/merge_request_widget_entity.rb b/app/serializers/merge_request_widget_entity.rb index 141070aef45..8260c6c7b84 100644 --- a/app/serializers/merge_request_widget_entity.rb +++ b/app/serializers/merge_request_widget_entity.rb @@ -13,7 +13,7 @@ class MergeRequestWidgetEntity < IssuableEntity expose :squash expose :target_branch expose :target_project_id - expose :allow_maintainer_to_push + expose :allow_collaboration expose :should_be_rebased?, as: :should_be_rebased expose :ff_only_enabled do |merge_request| diff --git a/app/services/merge_requests/base_service.rb b/app/services/merge_requests/base_service.rb index 231ab76fde4..4c420b38258 100644 --- a/app/services/merge_requests/base_service.rb +++ b/app/services/merge_requests/base_service.rb @@ -38,8 +38,8 @@ module MergeRequests def filter_params(merge_request) super - unless merge_request.can_allow_maintainer_to_push?(current_user) - params.delete(:allow_maintainer_to_push) + unless merge_request.can_allow_collaboration?(current_user) + params.delete(:allow_collaboration) end end diff --git a/app/views/shared/issuable/form/_contribution.html.haml b/app/views/shared/issuable/form/_contribution.html.haml index b34549240e0..519b5fae846 100644 --- a/app/views/shared/issuable/form/_contribution.html.haml +++ b/app/views/shared/issuable/form/_contribution.html.haml @@ -12,9 +12,9 @@ = _('Contribution') .col-sm-10 .form-check - = form.check_box :allow_maintainer_to_push, disabled: !issuable.can_allow_maintainer_to_push?(current_user), class: 'form-check-input' - = form.label :allow_maintainer_to_push, class: 'form-check-label' do - = _('Allow edits from maintainers.') - = link_to 'About this feature', help_page_path('user/project/merge_requests/maintainer_access') + = form.check_box :allow_collaboration, disabled: !issuable.can_allow_collaboration?(current_user), class: 'form-check-input' + = form.label :allow_collaboration, class: 'form-check-label' do + = _('Allow commits from members who can merge to the target branch.') + = link_to 'About this feature', help_page_path('user/project/merge_requests/allow_collaboration') .form-text.text-muted - = allow_maintainer_push_unavailable_reason(issuable) + = allow_collaboration_unavailable_reason(issuable) diff --git a/app/views/shared/projects/_edit_information.html.haml b/app/views/shared/projects/_edit_information.html.haml index ec9dc8f62c2..9230e045a81 100644 --- a/app/views/shared/projects/_edit_information.html.haml +++ b/app/views/shared/projects/_edit_information.html.haml @@ -1,6 +1,6 @@ - unless can?(current_user, :push_code, @project) .inline.prepend-left-10 - - if @project.branch_allows_maintainer_push?(current_user, selected_branch) + - if @project.branch_allows_collaboration?(current_user, selected_branch) = commit_in_single_accessible_branch - else = commit_in_fork_help diff --git a/changelogs/unreleased/42751-rename-mr-maintainer-push.yml b/changelogs/unreleased/42751-rename-mr-maintainer-push.yml new file mode 100644 index 00000000000..aa29f6ed4b7 --- /dev/null +++ b/changelogs/unreleased/42751-rename-mr-maintainer-push.yml @@ -0,0 +1,5 @@ +--- +title: Rephrasing Merge Request's 'allow edits from maintainer' functionality +merge_request: 19061 +author: +type: deprecated diff --git a/db/migrate/20180523042841_rename_merge_requests_allow_maintainer_to_push.rb b/db/migrate/20180523042841_rename_merge_requests_allow_maintainer_to_push.rb new file mode 100644 index 00000000000..975bdfe70f4 --- /dev/null +++ b/db/migrate/20180523042841_rename_merge_requests_allow_maintainer_to_push.rb @@ -0,0 +1,15 @@ +class RenameMergeRequestsAllowMaintainerToPush < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + rename_column_concurrently :merge_requests, :allow_maintainer_to_push, :allow_collaboration + end + + def down + cleanup_concurrent_column_rename :merge_requests, :allow_collaboration, :allow_maintainer_to_push + end +end diff --git a/db/post_migrate/20180523125103_cleanup_merge_requests_allow_maintainer_to_push_rename.rb b/db/post_migrate/20180523125103_cleanup_merge_requests_allow_maintainer_to_push_rename.rb new file mode 100644 index 00000000000..b9ce4600675 --- /dev/null +++ b/db/post_migrate/20180523125103_cleanup_merge_requests_allow_maintainer_to_push_rename.rb @@ -0,0 +1,15 @@ +class CleanupMergeRequestsAllowMaintainerToPushRename < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + cleanup_concurrent_column_rename :merge_requests, :allow_maintainer_to_push, :allow_collaboration + end + + def down + rename_column_concurrently :merge_requests, :allow_collaboration, :allow_maintainer_to_push + end +end diff --git a/db/schema.rb b/db/schema.rb index f8663574580..a53b09bfcb7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1216,7 +1216,7 @@ ActiveRecord::Schema.define(version: 20180529093006) do t.boolean "discussion_locked" t.integer "latest_merge_request_diff_id" t.string "rebase_commit_sha" - t.boolean "allow_maintainer_to_push" + t.boolean "allow_collaboration" t.boolean "squash", default: false, null: false end diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 051d2a10bc6..c31de9c3595 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -650,7 +650,8 @@ POST /projects/:id/merge_requests | `labels` | string | no | Labels for MR as a comma-separated list | | `milestone_id` | integer | no | The global ID of a milestone | | `remove_source_branch` | boolean | no | Flag indicating if a merge request should remove the source branch when merging | -| `allow_maintainer_to_push` | boolean | no | Whether or not a maintainer of the target project can push to the source branch | +| `allow_collaboration` | boolean | no | Allow commits from members who can merge to the target branch | +| `allow_maintainer_to_push` | boolean | no | Deprecated, see allow_collaboration | | `squash` | boolean | no | Squash commits into a single commit when merging | ```json @@ -708,6 +709,7 @@ POST /projects/:id/merge_requests "squash": false, "web_url": "http://example.com/example/example/merge_requests/1", "discussion_locked": false, + "allow_collaboration": false, "allow_maintainer_to_push": false, "time_stats": { "time_estimate": 0, @@ -740,7 +742,8 @@ PUT /projects/:id/merge_requests/:merge_request_iid | `remove_source_branch` | boolean | no | Flag indicating if a merge request should remove the source branch when merging | | `squash` | boolean | no | Squash commits into a single commit when merging | | `discussion_locked` | boolean | no | Flag indicating if the merge request's discussion is locked. If the discussion is locked only project members can add, edit or resolve comments. | -| `allow_maintainer_to_push` | boolean | no | Whether or not a maintainer of the target project can push to the source branch | +| `allow_collaboration` | boolean | no | Allow commits from members who can merge to the target branch | +| `allow_maintainer_to_push` | boolean | no | Deprecated, see allow_collaboration | Must include at least one non-required attribute from above. @@ -798,6 +801,7 @@ Must include at least one non-required attribute from above. "squash": false, "web_url": "http://example.com/example/example/merge_requests/1", "discussion_locked": false, + "allow_collaboration": false, "allow_maintainer_to_push": false, "time_stats": { "time_estimate": 0, diff --git a/doc/user/project/merge_requests/allow_collaboration.md b/doc/user/project/merge_requests/allow_collaboration.md new file mode 100644 index 00000000000..859ac92ef89 --- /dev/null +++ b/doc/user/project/merge_requests/allow_collaboration.md @@ -0,0 +1,20 @@ +# Allow collaboration on merge requests across forks + +> [Introduced][ce-17395] in GitLab 10.6. + +This feature is available for merge requests across forked projects that are +publicly accessible. It makes it easier for members of projects to +collaborate on merge requests across forks. + +When enabled for a merge request, members with merge access to the target +branch of the project will be granted write permissions to the source branch +of the merge request. + +The feature can only be enabled by users who already have push access to the +source project, and only lasts while the merge request is open. + +Enable this functionality while creating or editing a merge request: + +![Enable collaboration](./img/allow_collaboration.png) + +[ce-17395]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/17395 diff --git a/doc/user/project/merge_requests/img/allow_collaboration.png b/doc/user/project/merge_requests/img/allow_collaboration.png new file mode 100644 index 00000000000..75596e7d9ad Binary files /dev/null and b/doc/user/project/merge_requests/img/allow_collaboration.png differ diff --git a/doc/user/project/merge_requests/img/allow_maintainer_push.png b/doc/user/project/merge_requests/img/allow_maintainer_push.png deleted file mode 100644 index 91cc399f4ff..00000000000 Binary files a/doc/user/project/merge_requests/img/allow_maintainer_push.png and /dev/null differ diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index b75bcacc9d7..50979e82097 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -28,7 +28,7 @@ With GitLab merge requests, you can: - Enable [fast-forward merge requests](#fast-forward-merge-requests) - Enable [semi-linear history merge requests](#semi-linear-history-merge-requests) as another security layer to guarantee the pipeline is passing in the target branch - [Create new merge requests by email](#create-new-merge-requests-by-email) -- Allow maintainers of the target project to push directly to the fork by [allowing edits from maintainers](maintainer_access.md) +- [Allow collaboration](allow_collaboration.md) so members of the target project can push directly to the fork - [Squash and merge](squash_and_merge.md) for a cleaner commit history With **[GitLab Enterprise Edition][ee]**, you can also: diff --git a/doc/user/project/merge_requests/maintainer_access.md b/doc/user/project/merge_requests/maintainer_access.md index 89f71e16a50..d59afecd375 100644 --- a/doc/user/project/merge_requests/maintainer_access.md +++ b/doc/user/project/merge_requests/maintainer_access.md @@ -1,20 +1 @@ -# Allow maintainer pushes for merge requests across forks - -> [Introduced][ce-17395] in GitLab 10.6. - -This feature is available for merge requests across forked projects that are -publicly accessible. It makes it easier for maintainers of projects to -collaborate on merge requests across forks. - -When enabled for a merge request, members with merge access to the target -branch of the project will be granted write permissions to the source branch -of the merge request. - -The feature can only be enabled by users who already have push access to the -source project, and only lasts while the merge request is open. - -Enable this functionality while creating a merge request: - -![Enable maintainer edits](./img/allow_maintainer_push.png) - -[ce-17395]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/17395 +This document was moved to [another location](allow_collaboration.md). diff --git a/lib/api/entities.rb b/lib/api/entities.rb index c4537036a3a..c76d3ff45d0 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -559,7 +559,9 @@ module API expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch - expose :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } + expose :allow_collaboration, if: -> (merge_request, _) { merge_request.for_fork? } + # Deprecated + expose :allow_collaboration, as: :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } expose :web_url do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index b1e510d72de..55e83303536 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -162,7 +162,8 @@ module API optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' optional :labels, type: String, desc: 'Comma-separated list of label names' optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' - optional :allow_maintainer_to_push, type: Boolean, desc: 'Whether a maintainer of the target project can push to the source project' + optional :allow_collaboration, type: Boolean, desc: 'Allow commits from members who can merge to the target branch' + optional :allow_maintainer_to_push, type: Boolean, as: :allow_collaboration, desc: '[deprecated] See allow_collaboration' optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge' use :optional_params_ee diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 8cf5d636743..27560abfb96 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -65,7 +65,7 @@ module Gitlab return false unless can_access_git? return false unless project - return false if !user.can?(:push_code, project) && !project.branch_allows_maintainer_push?(user, ref) + return false if !user.can?(:push_code, project) && !project.branch_allows_collaboration?(user, ref) if protected?(ProtectedBranch, project, ref) protected_branch_accessible_to?(ref, action: :push) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 035a2275d9f..8ae04cd2f88 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -331,7 +331,7 @@ msgstr "" msgid "All features are enabled for blank projects, from templates, or when importing, but you can disable them afterward in the project settings." msgstr "" -msgid "Allow edits from maintainers." +msgid "Allow commits from members who can merge to the target branch." msgstr "" msgid "Allow rendering of PlantUML diagrams in Asciidoc documents." @@ -4894,7 +4894,7 @@ msgstr "" msgid "mrWidget|%{metricsLinkStart} Memory %{metricsLinkEnd} usage is %{emphasisStart} unchanged %{emphasisEnd} at %{memoryFrom}MB" msgstr "" -msgid "mrWidget|Allows edits from maintainers" +msgid "mrWidget|Allows commits from members who can merge to the target branch" msgstr "" msgid "mrWidget|Cancel automatic merge" diff --git a/spec/features/merge_request/maintainer_edits_fork_spec.rb b/spec/features/merge_request/maintainer_edits_fork_spec.rb index a3323da1b1f..1808d0c0a0c 100644 --- a/spec/features/merge_request/maintainer_edits_fork_spec.rb +++ b/spec/features/merge_request/maintainer_edits_fork_spec.rb @@ -14,7 +14,7 @@ describe 'a maintainer edits files on a source-branch of an MR from a fork', :js source_branch: 'fix', target_branch: 'master', author: author, - allow_maintainer_to_push: true) + allow_collaboration: true) end before do diff --git a/spec/features/merge_request/user_allows_a_maintainer_to_push_spec.rb b/spec/features/merge_request/user_allows_a_maintainer_to_push_spec.rb deleted file mode 100644 index eb41d7de8ed..00000000000 --- a/spec/features/merge_request/user_allows_a_maintainer_to_push_spec.rb +++ /dev/null @@ -1,85 +0,0 @@ -require 'spec_helper' - -describe 'create a merge request that allows maintainers to push', :js do - include ProjectForksHelper - let(:user) { create(:user) } - let(:target_project) { create(:project, :public, :repository) } - let(:source_project) { fork_project(target_project, user, repository: true, namespace: user.namespace) } - - def visit_new_merge_request - visit project_new_merge_request_path( - source_project, - merge_request: { - source_project_id: source_project.id, - target_project_id: target_project.id, - source_branch: 'fix', - target_branch: 'master' - }) - end - - before do - sign_in(user) - end - - it 'allows setting maintainer push possible' do - visit_new_merge_request - - check 'Allow edits from maintainers' - - click_button 'Submit merge request' - - wait_for_requests - - expect(page).to have_content('Allows edits from maintainers') - end - - it 'shows a message when one of the projects is private' do - source_project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE) - - visit_new_merge_request - - expect(page).to have_content('Not available for private projects') - end - - it 'shows a message when the source branch is protected' do - create(:protected_branch, project: source_project, name: 'fix') - - visit_new_merge_request - - expect(page).to have_content('Not available for protected branches') - end - - context 'when the merge request is being created within the same project' do - let(:source_project) { target_project } - - it 'hides the checkbox if the merge request is being created within the same project' do - target_project.add_developer(user) - - visit_new_merge_request - - expect(page).not_to have_content('Allows edits from maintainers') - end - end - - context 'when a maintainer tries to edit the option' do - let(:maintainer) { create(:user) } - let(:merge_request) do - create(:merge_request, - source_project: source_project, - target_project: target_project, - source_branch: 'fixes') - end - - before do - target_project.add_master(maintainer) - - sign_in(maintainer) - end - - it 'it hides the option from maintainers' do - visit edit_project_merge_request_path(target_project, merge_request) - - expect(page).not_to have_content('Allows edits from maintainers') - end - end -end diff --git a/spec/features/merge_request/user_allows_commits_from_memebers_who_can_merge_spec.rb b/spec/features/merge_request/user_allows_commits_from_memebers_who_can_merge_spec.rb new file mode 100644 index 00000000000..0af37d76539 --- /dev/null +++ b/spec/features/merge_request/user_allows_commits_from_memebers_who_can_merge_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +describe 'create a merge request, allowing commits from members who can merge to the target branch', :js do + include ProjectForksHelper + let(:user) { create(:user) } + let(:target_project) { create(:project, :public, :repository) } + let(:source_project) { fork_project(target_project, user, repository: true, namespace: user.namespace) } + + def visit_new_merge_request + visit project_new_merge_request_path( + source_project, + merge_request: { + source_project_id: source_project.id, + target_project_id: target_project.id, + source_branch: 'fix', + target_branch: 'master' + }) + end + + before do + sign_in(user) + end + + it 'allows setting possible' do + visit_new_merge_request + + check 'Allow commits from members who can merge to the target branch' + + click_button 'Submit merge request' + + wait_for_requests + + expect(page).to have_content('Allows commits from members who can merge to the target branch') + end + + it 'shows a message when one of the projects is private' do + source_project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE) + + visit_new_merge_request + + expect(page).to have_content('Not available for private projects') + end + + it 'shows a message when the source branch is protected' do + create(:protected_branch, project: source_project, name: 'fix') + + visit_new_merge_request + + expect(page).to have_content('Not available for protected branches') + end + + context 'when the merge request is being created within the same project' do + let(:source_project) { target_project } + + it 'hides the checkbox if the merge request is being created within the same project' do + target_project.add_developer(user) + + visit_new_merge_request + + expect(page).not_to have_content('Allows commits from members who can merge to the target branch') + end + end + + context 'when a member who can merge tries to edit the option' do + let(:member) { create(:user) } + let(:merge_request) do + create(:merge_request, + source_project: source_project, + target_project: target_project, + source_branch: 'fixes') + end + + before do + target_project.add_master(member) + + sign_in(member) + end + + it 'it hides the option from members' do + visit edit_project_merge_request_path(target_project, merge_request) + + expect(page).not_to have_content('Allows commits from members who can merge to the target branch') + end + end +end diff --git a/spec/fixtures/api/schemas/entities/merge_request_basic.json b/spec/fixtures/api/schemas/entities/merge_request_basic.json index 46031961cca..f7bc137c90c 100644 --- a/spec/fixtures/api/schemas/entities/merge_request_basic.json +++ b/spec/fixtures/api/schemas/entities/merge_request_basic.json @@ -13,6 +13,7 @@ "assignee_id": { "type": ["integer", "null"] }, "subscribed": { "type": ["boolean", "null"] }, "participants": { "type": "array" }, + "allow_collaboration": { "type": "boolean"}, "allow_maintainer_to_push": { "type": "boolean"} }, "additionalProperties": false diff --git a/spec/fixtures/api/schemas/entities/merge_request_widget.json b/spec/fixtures/api/schemas/entities/merge_request_widget.json index 7be8c9e3e67..ee5588fa6c6 100644 --- a/spec/fixtures/api/schemas/entities/merge_request_widget.json +++ b/spec/fixtures/api/schemas/entities/merge_request_widget.json @@ -31,7 +31,7 @@ "source_project_id": { "type": "integer" }, "target_branch": { "type": "string" }, "target_project_id": { "type": "integer" }, - "allow_maintainer_to_push": { "type": "boolean"}, + "allow_collaboration": { "type": "boolean"}, "metrics": { "oneOf": [ { "type": "null" }, diff --git a/spec/fixtures/api/schemas/public_api/v4/merge_requests.json b/spec/fixtures/api/schemas/public_api/v4/merge_requests.json index f97461ce9cc..f7adc4e0b91 100644 --- a/spec/fixtures/api/schemas/public_api/v4/merge_requests.json +++ b/spec/fixtures/api/schemas/public_api/v4/merge_requests.json @@ -82,6 +82,7 @@ "human_time_estimate": { "type": ["string", "null"] }, "human_total_time_spent": { "type": ["string", "null"] } }, + "allow_collaboration": { "type": ["boolean", "null"] }, "allow_maintainer_to_push": { "type": ["boolean", "null"] } }, "required": [ diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 74e7a45fd6c..d389dcb4e80 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -170,7 +170,7 @@ MergeRequest: - last_edited_by_id - head_pipeline_id - discussion_locked -- allow_maintainer_to_push +- allow_collaboration MergeRequestDiff: - id - state diff --git a/spec/lib/gitlab/user_access_spec.rb b/spec/lib/gitlab/user_access_spec.rb index 97b6069f64d..0469d984a40 100644 --- a/spec/lib/gitlab/user_access_spec.rb +++ b/spec/lib/gitlab/user_access_spec.rb @@ -142,7 +142,7 @@ describe Gitlab::UserAccess do target_project: canonical_project, source_project: project, source_branch: 'awesome-feature', - allow_maintainer_to_push: true + allow_collaboration: true ) end diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 9ffa91fc265..8718fe50769 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -2236,25 +2236,25 @@ describe MergeRequest do end end - describe '#allow_maintainer_to_push' do + describe '#allow_collaboration' do let(:merge_request) do - build(:merge_request, source_branch: 'fixes', allow_maintainer_to_push: true) + build(:merge_request, source_branch: 'fixes', allow_collaboration: true) end it 'is false when pushing by a maintainer is not possible' do - expect(merge_request).to receive(:maintainer_push_possible?) { false } + expect(merge_request).to receive(:collaborative_push_possible?) { false } - expect(merge_request.allow_maintainer_to_push).to be_falsy + expect(merge_request.allow_collaboration).to be_falsy end it 'is true when pushing by a maintainer is possible' do - expect(merge_request).to receive(:maintainer_push_possible?) { true } + expect(merge_request).to receive(:collaborative_push_possible?) { true } - expect(merge_request.allow_maintainer_to_push).to be_truthy + expect(merge_request.allow_collaboration).to be_truthy end end - describe '#maintainer_push_possible?' do + describe '#collaborative_push_possible?' do let(:merge_request) do build(:merge_request, source_branch: 'fixes') end @@ -2266,14 +2266,14 @@ describe MergeRequest do it 'does not allow maintainer to push if the source project is the same as the target' do merge_request.target_project = merge_request.source_project = create(:project, :public) - expect(merge_request.maintainer_push_possible?).to be_falsy + expect(merge_request.collaborative_push_possible?).to be_falsy end it 'allows maintainer to push when both source and target are public' do merge_request.target_project = build(:project, :public) merge_request.source_project = build(:project, :public) - expect(merge_request.maintainer_push_possible?).to be_truthy + expect(merge_request.collaborative_push_possible?).to be_truthy end it 'is not available for protected branches' do @@ -2284,11 +2284,11 @@ describe MergeRequest do .with(merge_request.source_project, 'fixes') .and_return(true) - expect(merge_request.maintainer_push_possible?).to be_falsy + expect(merge_request.collaborative_push_possible?).to be_falsy end end - describe '#can_allow_maintainer_to_push?' do + describe '#can_allow_collaboration?' do let(:target_project) { create(:project, :public) } let(:source_project) { fork_project(target_project) } let(:merge_request) do @@ -2300,17 +2300,17 @@ describe MergeRequest do let(:user) { create(:user) } before do - allow(merge_request).to receive(:maintainer_push_possible?) { true } + allow(merge_request).to receive(:collaborative_push_possible?) { true } end it 'is false if the user does not have push access to the source project' do - expect(merge_request.can_allow_maintainer_to_push?(user)).to be_falsy + expect(merge_request.can_allow_collaboration?(user)).to be_falsy end it 'is true when the user has push access to the source project' do source_project.add_developer(user) - expect(merge_request.can_allow_maintainer_to_push?(user)).to be_truthy + expect(merge_request.can_allow_collaboration?(user)).to be_truthy end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index af2240f4f89..b0cbf8796e3 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3583,7 +3583,7 @@ describe Project do target_branch: 'target-branch', source_project: project, source_branch: 'awesome-feature-1', - allow_maintainer_to_push: true + allow_collaboration: true ) end @@ -3620,9 +3620,9 @@ describe Project do end end - describe '#branch_allows_maintainer_push?' do + describe '#branch_allows_collaboration_push?' do it 'allows access if the user can merge the merge request' do - expect(project.branch_allows_maintainer_push?(user, 'awesome-feature-1')) + expect(project.branch_allows_collaboration?(user, 'awesome-feature-1')) .to be_truthy end @@ -3630,7 +3630,7 @@ describe Project do guest = create(:user) target_project.add_guest(guest) - expect(project.branch_allows_maintainer_push?(guest, 'awesome-feature-1')) + expect(project.branch_allows_collaboration?(guest, 'awesome-feature-1')) .to be_falsy end @@ -3640,31 +3640,31 @@ describe Project do target_branch: 'target-branch', source_project: project, source_branch: 'rejected-feature-1', - allow_maintainer_to_push: true) + allow_collaboration: true) - expect(project.branch_allows_maintainer_push?(user, 'rejected-feature-1')) + expect(project.branch_allows_collaboration?(user, 'rejected-feature-1')) .to be_falsy end it 'does not allow access if the user cannot merge the merge request' do create(:protected_branch, :masters_can_push, project: target_project, name: 'target-branch') - expect(project.branch_allows_maintainer_push?(user, 'awesome-feature-1')) + expect(project.branch_allows_collaboration?(user, 'awesome-feature-1')) .to be_falsy end it 'caches the result' do - control = ActiveRecord::QueryRecorder.new { project.branch_allows_maintainer_push?(user, 'awesome-feature-1') } + control = ActiveRecord::QueryRecorder.new { project.branch_allows_collaboration?(user, 'awesome-feature-1') } - expect { 3.times { project.branch_allows_maintainer_push?(user, 'awesome-feature-1') } } + expect { 3.times { project.branch_allows_collaboration?(user, 'awesome-feature-1') } } .not_to exceed_query_limit(control) end context 'when the requeststore is active', :request_store do it 'only queries per project across instances' do - control = ActiveRecord::QueryRecorder.new { project.branch_allows_maintainer_push?(user, 'awesome-feature-1') } + control = ActiveRecord::QueryRecorder.new { project.branch_allows_collaboration?(user, 'awesome-feature-1') } - expect { 2.times { described_class.find(project.id).branch_allows_maintainer_push?(user, 'awesome-feature-1') } } + expect { 2.times { described_class.find(project.id).branch_allows_collaboration?(user, 'awesome-feature-1') } } .not_to exceed_query_limit(control).with_threshold(2) end end diff --git a/spec/policies/ci/build_policy_spec.rb b/spec/policies/ci/build_policy_spec.rb index 9ca156deaa0..eead55d33ca 100644 --- a/spec/policies/ci/build_policy_spec.rb +++ b/spec/policies/ci/build_policy_spec.rb @@ -101,7 +101,7 @@ describe Ci::BuildPolicy do it 'enables update_build if user is maintainer' do allow_any_instance_of(Project).to receive(:empty_repo?).and_return(false) - allow_any_instance_of(Project).to receive(:branch_allows_maintainer_push?).and_return(true) + allow_any_instance_of(Project).to receive(:branch_allows_collaboration?).and_return(true) expect(policy).to be_allowed :update_build expect(policy).to be_allowed :update_commit_status diff --git a/spec/policies/ci/pipeline_policy_spec.rb b/spec/policies/ci/pipeline_policy_spec.rb index a5e509cfa0f..bd32faf06ef 100644 --- a/spec/policies/ci/pipeline_policy_spec.rb +++ b/spec/policies/ci/pipeline_policy_spec.rb @@ -69,7 +69,7 @@ describe Ci::PipelinePolicy, :models do it 'enables update_pipeline if user is maintainer' do allow_any_instance_of(Project).to receive(:empty_repo?).and_return(false) - allow_any_instance_of(Project).to receive(:branch_allows_maintainer_push?).and_return(true) + allow_any_instance_of(Project).to receive(:branch_allows_collaboration?).and_return(true) expect(policy).to be_allowed :update_pipeline end diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb index 6609f5f7afd..6ac151f92f3 100644 --- a/spec/policies/project_policy_spec.rb +++ b/spec/policies/project_policy_spec.rb @@ -400,7 +400,7 @@ describe ProjectPolicy do :merge_request, target_project: target_project, source_project: project, - allow_maintainer_to_push: true + allow_collaboration: true ) end let(:maintainer_abilities) do diff --git a/spec/requests/api/merge_requests_spec.rb b/spec/requests/api/merge_requests_spec.rb index 605761867bf..d4ebfc3f782 100644 --- a/spec/requests/api/merge_requests_spec.rb +++ b/spec/requests/api/merge_requests_spec.rb @@ -386,12 +386,13 @@ describe API::MergeRequests do source_project: forked_project, target_project: project, source_branch: 'fixes', - allow_maintainer_to_push: true) + allow_collaboration: true) end - it 'includes the `allow_maintainer_to_push` field' do + it 'includes the `allow_collaboration` field' do get api("/projects/#{project.id}/merge_requests/#{merge_request.iid}", user) + expect(json_response['allow_collaboration']).to be_truthy expect(json_response['allow_maintainer_to_push']).to be_truthy end end @@ -654,11 +655,12 @@ describe API::MergeRequests do expect(response).to have_gitlab_http_status(400) end - it 'allows setting `allow_maintainer_to_push`' do + it 'allows setting `allow_collaboration`' do post api("/projects/#{forked_project.id}/merge_requests", user2), - title: 'Test merge_request', source_branch: "feature_conflict", target_branch: "master", - author: user2, target_project_id: project.id, allow_maintainer_to_push: true + title: 'Test merge_request', source_branch: "feature_conflict", target_branch: "master", + author: user2, target_project_id: project.id, allow_collaboration: true expect(response).to have_gitlab_http_status(201) + expect(json_response['allow_collaboration']).to be_truthy expect(json_response['allow_maintainer_to_push']).to be_truthy end diff --git a/spec/services/ci/retry_pipeline_service_spec.rb b/spec/services/ci/retry_pipeline_service_spec.rb index a73bd7a0268..688d3b8c038 100644 --- a/spec/services/ci/retry_pipeline_service_spec.rb +++ b/spec/services/ci/retry_pipeline_service_spec.rb @@ -280,12 +280,12 @@ describe Ci::RetryPipelineService, '#execute' do source_project: forked_project, target_project: project, source_branch: 'fixes', - allow_maintainer_to_push: true) + allow_collaboration: true) create_build('rspec 1', :failed, 1) end it 'allows to retry failed pipeline' do - allow_any_instance_of(Project).to receive(:fetch_branch_allows_maintainer_push?).and_return(true) + allow_any_instance_of(Project).to receive(:fetch_branch_allows_collaboration?).and_return(true) allow_any_instance_of(Project).to receive(:empty_repo?).and_return(false) service.execute(pipeline) diff --git a/spec/services/merge_requests/update_service_spec.rb b/spec/services/merge_requests/update_service_spec.rb index 5279ea6164e..cc26df725c7 100644 --- a/spec/services/merge_requests/update_service_spec.rb +++ b/spec/services/merge_requests/update_service_spec.rb @@ -541,7 +541,7 @@ describe MergeRequests::UpdateService, :mailer do let(:closed_issuable) { create(:closed_merge_request, source_project: project) } end - context 'setting `allow_maintainer_to_push`' do + context 'setting `allow_collaboration`' do let(:target_project) { create(:project, :public) } let(:source_project) { fork_project(target_project) } let(:user) { create(:user) } @@ -556,23 +556,23 @@ describe MergeRequests::UpdateService, :mailer do allow(ProtectedBranch).to receive(:protected?).with(source_project, 'fixes') { false } end - it 'does not allow a maintainer of the target project to set `allow_maintainer_to_push`' do + it 'does not allow a maintainer of the target project to set `allow_collaboration`' do target_project.add_developer(user) - update_merge_request(allow_maintainer_to_push: true, title: 'Updated title') + update_merge_request(allow_collaboration: true, title: 'Updated title') expect(merge_request.title).to eq('Updated title') - expect(merge_request.allow_maintainer_to_push).to be_falsy + expect(merge_request.allow_collaboration).to be_falsy end it 'is allowed by a user that can push to the source and can update the merge request' do merge_request.update!(assignee: user) source_project.add_developer(user) - update_merge_request(allow_maintainer_to_push: true, title: 'Updated title') + update_merge_request(allow_collaboration: true, title: 'Updated title') expect(merge_request.title).to eq('Updated title') - expect(merge_request.allow_maintainer_to_push).to be_truthy + expect(merge_request.allow_collaboration).to be_truthy end end end -- cgit v1.2.1 From 4beeb60255f228dc45dbe8f675a3cc59c0ea7773 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Jun 2018 14:36:52 +0900 Subject: Fix populate_spec --- spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index bcfa9f0c282..feed7728f5a 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -75,7 +75,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do it 'wastes pipeline iid' do expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0 - end + end end context 'when pipeline has validation errors' do @@ -98,7 +98,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do it 'wastes pipeline iid' do expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0 - end + end end context 'when there is a seed blocks present' do @@ -144,7 +144,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do step.perform rescue nil expect(InternalId.ci_pipelines.where(project_id: project.id).exists?).to be_falsy - end + end end end -- cgit v1.2.1 From f7f60ab54ab69fb4d0c3a43406a9809edab7d762 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Jun 2018 14:53:00 +0900 Subject: Add spec for variables expressions with pipeline iid --- spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb | 45 +++++++++++++++------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index feed7728f5a..6b18c615430 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -156,22 +156,41 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do end end - context 'when using only/except build policies' do - let(:config) do - { rspec: { script: 'rspec', stage: 'test', only: ['master'] }, - prod: { script: 'cap prod', stage: 'deploy', only: ['tags'] } } - end + context 'when variables policy is specified' do + context 'when using only/except build policies' do + let(:config) do + { rspec: { script: 'rspec', stage: 'test', only: ['master'] }, + prod: { script: 'cap prod', stage: 'deploy', only: ['tags'] } } + end - let(:pipeline) do - build(:ci_pipeline, ref: 'master', config: config) - end + let(:pipeline) do + build(:ci_pipeline, ref: 'master', config: config) + end - it 'populates pipeline according to used policies' do - step.perform! + it 'populates pipeline according to used policies' do + step.perform! - expect(pipeline.stages.size).to eq 1 - expect(pipeline.stages.first.builds.size).to eq 1 - expect(pipeline.stages.first.builds.first.name).to eq 'rspec' + expect(pipeline.stages.size).to eq 1 + expect(pipeline.stages.first.builds.size).to eq 1 + expect(pipeline.stages.first.builds.first.name).to eq 'rspec' + end + + context 'when variables expression is specified' do + let(:config) do + { rspec: { script: 'rspec', only: { variables: ["$CI_PIPELINE_IID == '1'"] } }, + prod: { script: 'cap prod', only: { variables: ["$CI_PIPELINE_IID == '1000'"] } } } + end + + context 'when pipeline iid is the subject' do + it 'populates pipeline according to used policies' do + step.perform! + + expect(pipeline.stages.size).to eq 1 + expect(pipeline.stages.first.builds.size).to eq 1 + expect(pipeline.stages.first.builds.first.name).to eq 'rspec' + end + end + end end end end -- cgit v1.2.1 From c754b6937c8077304386b3a6b37233e52eacdb3e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Jun 2018 15:37:36 +0900 Subject: Clean up presence validation spec --- spec/models/ci/pipeline_spec.rb | 3 +-- .../models/atomic_internal_id_spec.rb | 28 +++++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 7d28f2eb86b..e03c068b88e 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -36,13 +36,12 @@ describe Ci::Pipeline, :mailer do end describe 'modules' do - it_behaves_like 'AtomicInternalId' do + it_behaves_like 'AtomicInternalId', validate_presence: false do let(:internal_id_attribute) { :iid } let(:instance) { build(:ci_pipeline) } let(:scope) { :project } let(:scope_attrs) { { project: instance.project } } let(:usage) { :ci_pipelines } - let(:allow_nil) { true } end end diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb index a05279364f2..d0cd8da67e1 100644 --- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb +++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb @@ -1,8 +1,6 @@ require 'spec_helper' -shared_examples_for 'AtomicInternalId' do - let(:allow_nil) { false } - +shared_examples_for 'AtomicInternalId' do |validate_presence: true| describe '.has_internal_id' do describe 'Module inclusion' do subject { described_class } @@ -12,18 +10,30 @@ shared_examples_for 'AtomicInternalId' do describe 'Validation' do before do - allow_any_instance_of(described_class).to receive(:"ensure_#{scope}_#{internal_id_attribute}!") {} - end + allow_any_instance_of(described_class).to receive(:"ensure_#{scope}_#{internal_id_attribute}!") - it 'validates presence' do instance.valid? + end - if allow_nil - expect(instance.errors[internal_id_attribute]).to be_empty - else + context 'when presence validattion is required' do + before do + skip unless validate_presence + end + + it 'validates presence' do expect(instance.errors[internal_id_attribute]).to include("can't be blank") end end + + context 'when presence validattion is not required' do + before do + skip if validate_presence + end + + it 'does not validate presence' do + expect(instance.errors[internal_id_attribute]).to be_empty + end + end end describe 'Creating an instance' do -- cgit v1.2.1 From c418d68765eb09c468419ec8f438100cda64a0d4 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Jun 2018 15:41:33 +0900 Subject: Remove unneccesary spec --- spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb | 2 +- spec/models/ci/pipeline_spec.rb | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index 6b18c615430..ffb2c1d5b0c 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -188,7 +188,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do expect(pipeline.stages.size).to eq 1 expect(pipeline.stages.first.builds.size).to eq 1 expect(pipeline.stages.first.builds.first.name).to eq 'rspec' - end + end end end end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index e03c068b88e..2b9c232743d 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -396,20 +396,6 @@ describe Ci::Pipeline, :mailer do expect(seeds.size).to eq 1 expect(seeds.dig(0, 0, :name)).to eq 'unit' end - - context "when pipeline iid is used for 'only' keyword" do - let(:config) do - { rspec: { script: 'rspec', only: { variables: ['$CI_PIPELINE_IID == 2'] } }, - prod: { script: 'cap prod', only: { variables: ['$CI_PIPELINE_IID == 1'] } } } - end - - it 'returns stage seeds only when variables expression is truthy' do - seeds = pipeline.stage_seeds - - expect(seeds.size).to eq 1 - expect(seeds.dig(0, 0, :name)).to eq 'prod' - end - end end end -- cgit v1.2.1 From c89e57842ebf7f395363bcddaeff76bc7b3f7890 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Jun 2018 15:46:15 +0900 Subject: Use shared examples for populate spec --- spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb | 34 ++++++++++------------ .../models/atomic_internal_id_spec.rb | 4 +-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index ffb2c1d5b0c..7088233f237 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -157,6 +157,16 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do end context 'when variables policy is specified' do + shared_examples_for 'populates pipeline according to used policies' do + it 'populates pipeline according to used policies' do + step.perform! + + expect(pipeline.stages.size).to eq 1 + expect(pipeline.stages.first.builds.size).to eq 1 + expect(pipeline.stages.first.builds.first.name).to eq 'rspec' + end + end + context 'when using only/except build policies' do let(:config) do { rspec: { script: 'rspec', stage: 'test', only: ['master'] }, @@ -167,28 +177,16 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do build(:ci_pipeline, ref: 'master', config: config) end - it 'populates pipeline according to used policies' do - step.perform! - - expect(pipeline.stages.size).to eq 1 - expect(pipeline.stages.first.builds.size).to eq 1 - expect(pipeline.stages.first.builds.first.name).to eq 'rspec' - end + it_behaves_like 'populates pipeline according to used policies' context 'when variables expression is specified' do - let(:config) do - { rspec: { script: 'rspec', only: { variables: ["$CI_PIPELINE_IID == '1'"] } }, - prod: { script: 'cap prod', only: { variables: ["$CI_PIPELINE_IID == '1000'"] } } } - end - context 'when pipeline iid is the subject' do - it 'populates pipeline according to used policies' do - step.perform! - - expect(pipeline.stages.size).to eq 1 - expect(pipeline.stages.first.builds.size).to eq 1 - expect(pipeline.stages.first.builds.first.name).to eq 'rspec' + let(:config) do + { rspec: { script: 'rspec', only: { variables: ["$CI_PIPELINE_IID == '1'"] } }, + prod: { script: 'cap prod', only: { variables: ["$CI_PIPELINE_IID == '1000'"] } } } end + + it_behaves_like 'populates pipeline according to used policies' end end end diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb index d0cd8da67e1..7ab1041d17c 100644 --- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb +++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb @@ -15,7 +15,7 @@ shared_examples_for 'AtomicInternalId' do |validate_presence: true| instance.valid? end - context 'when presence validattion is required' do + context 'when presence validation is required' do before do skip unless validate_presence end @@ -25,7 +25,7 @@ shared_examples_for 'AtomicInternalId' do |validate_presence: true| end end - context 'when presence validattion is not required' do + context 'when presence validation is not required' do before do skip if validate_presence end -- cgit v1.2.1 From 60c4f2840ed632ea2ae154d42992e84357d238c9 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Fri, 1 Jun 2018 15:19:46 +0100 Subject: Update to GitLab Workhorse v4.3.0 --- GITLAB_WORKHORSE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION index fae6e3d04b2..80895903a15 100644 --- a/GITLAB_WORKHORSE_VERSION +++ b/GITLAB_WORKHORSE_VERSION @@ -1 +1 @@ -4.2.1 +4.3.0 -- cgit v1.2.1 From e5dddae1c597a47bb476df476c445f928e279183 Mon Sep 17 00:00:00 2001 From: Phil Hughes Date: Fri, 1 Jun 2018 14:40:52 +0000 Subject: Fixed typo in commit message help popover Closes #47060 --- app/assets/javascripts/ide/components/commit_sidebar/message_field.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/ide/components/commit_sidebar/message_field.vue b/app/assets/javascripts/ide/components/commit_sidebar/message_field.vue index f14fcdc88ed..0ac0af2feaa 100644 --- a/app/assets/javascripts/ide/components/commit_sidebar/message_field.vue +++ b/app/assets/javascripts/ide/components/commit_sidebar/message_field.vue @@ -54,7 +54,7 @@ export default { placement: 'top', content: sprintf( __(` - The character highligher helps you keep the subject line to %{titleLength} characters + The character highlighter helps you keep the subject line to %{titleLength} characters and wrap the body at %{bodyLength} so they are readable in git. `), { titleLength: MAX_TITLE_LENGTH, bodyLength: MAX_BODY_LENGTH }, -- cgit v1.2.1 From afcc0784eac952d15210a798e5b57f0e7eb82eee Mon Sep 17 00:00:00 2001 From: Olivier Gonzalez Date: Fri, 1 Jun 2018 11:10:59 -0400 Subject: Update security products job and artifact names in documentation. Refs gitlab-org/gitlab-ee#6184 --- doc/topics/autodevops/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index efec365042a..1400b2e36fe 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -497,10 +497,10 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac | `CANARY_ENABLED` | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). | | `INCREMENTAL_ROLLOUT_ENABLED`| From GitLab 10.8, this variable can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. | | `TEST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `test` job. If the variable is present, the job will not be created. | -| `CODEQUALITY_DISABLED` | From GitLab 11.0, this variable can be used to disable the `codequality` job. If the variable is present, the job will not be created. | +| `CODE_QUALITY_DISABLED` | From GitLab 11.0, this variable can be used to disable the `code_quality` job. If the variable is present, the job will not be created. | | `SAST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `sast` job. If the variable is present, the job will not be created. | | `DEPENDENCY_SCANNING_DISABLED` | From GitLab 11.0, this variable can be used to disable the `dependency_scanning` job. If the variable is present, the job will not be created. | -| `CONTAINER_SCANNING_DISABLED` | From GitLab 11.0, this variable can be used to disable the `sast:container` job. If the variable is present, the job will not be created. | +| `CONTAINER_SCANNING_DISABLED` | From GitLab 11.0, this variable can be used to disable the `container_scanning` job. If the variable is present, the job will not be created. | | `REVIEW_DISABLED` | From GitLab 11.0, this variable can be used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs will not be created. | | `DAST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `dast` job. If the variable is present, the job will not be created. | | `PERFORMANCE_DISABLED` | From GitLab 11.0, this variable can be used to disable the `performance` job. If the variable is present, the job will not be created. | -- cgit v1.2.1 From e1ffd6a271e352a725aa0394b654e9250f2d8240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 31 May 2018 18:12:48 +0200 Subject: Use RequestStore to memoize Flipper features so that memoized values are cleared between requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/feature.rb | 11 +++++++++-- spec/lib/feature_spec.rb | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/lib/feature.rb b/lib/feature.rb index 6474de6e56d..314ae224d90 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -63,8 +63,15 @@ class Feature end def flipper - Thread.current[:flipper] ||= - Flipper.new(flipper_adapter).tap { |flip| flip.memoize = true } + if RequestStore.active? + RequestStore[:flipper] ||= build_flipper_instance + else + @flipper ||= build_flipper_instance + end + end + + def build_flipper_instance + Flipper.new(flipper_adapter).tap { |flip| flip.memoize = true } end # This method is called from config/initializers/flipper.rb and can be used diff --git a/spec/lib/feature_spec.rb b/spec/lib/feature_spec.rb index 10020511bf8..6eb10497428 100644 --- a/spec/lib/feature_spec.rb +++ b/spec/lib/feature_spec.rb @@ -64,4 +64,28 @@ describe Feature do expect(described_class.all).to eq(features.to_a) end end + + describe '.flipper' do + shared_examples 'a memoized Flipper instance' do + it 'memoizes the Flipper instance' do + expect(Flipper).to receive(:new).once.and_call_original + + 2.times do + described_class.flipper + end + end + end + + context 'when request store is inactive' do + before do + described_class.instance_variable_set(:@flipper, nil) + end + + it_behaves_like 'a memoized Flipper instance' + end + + context 'when request store is inactive', :request_store do + it_behaves_like 'a memoized Flipper instance' + end + end end -- cgit v1.2.1 From c9cf677e20737aa9f8a824ec2e51b4ed12bc8ea4 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 1 Jun 2018 11:18:18 -0700 Subject: Remove ambiguity in Group class causing build failures https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/71887977 was failing due to confusion between Ci::Group and Group. https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/71887977 --- app/services/ci/register_job_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 317d1defbba..925775aea0b 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -90,7 +90,7 @@ module Ci def builds_for_group_runner # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL` - groups = Group.joins(:runner_namespaces).merge(runner.runner_namespaces) + groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces) hierarchy_groups = Gitlab::GroupHierarchy.new(groups).base_and_descendants projects = Project.where(namespace_id: hierarchy_groups) -- cgit v1.2.1 From 78d78ad1991f0a27b8ff79614d09f85909d20ed1 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 1 Jun 2018 13:44:16 -0700 Subject: Add comment about the need for truncating keys in Ruby 2.4 [ci skip] --- config/settings.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/settings.rb b/config/settings.rb index 4aa903109ea..58f38d103ea 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -85,7 +85,14 @@ class Settings < Settingslogic File.expand_path(path, Rails.root) end + # Returns a 256-bit key for attr_encrypted def attr_encrypted_db_key_base + # Ruby 2.4+ requires passing in the exact required length for OpenSSL keys + # (https://github.com/ruby/ruby/commit/ce635262f53b760284d56bb1027baebaaec175d1). + # Previous versions quietly truncated the input. + # + # The default mode for the attr_encrypted gem is to use a 256-bit key. + # We truncate the 128-byte string to 32 bytes. Gitlab::Application.secrets.db_key_base[0..31] end -- cgit v1.2.1 From eb1324dbb7f6cda34d902cd5ead2841bc3c674c9 Mon Sep 17 00:00:00 2001 From: Clement Ho Date: Fri, 1 Jun 2018 21:58:00 +0000 Subject: Fix bootstrap 4 file inputs --- app/assets/stylesheets/bootstrap_migration.scss | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/assets/stylesheets/bootstrap_migration.scss b/app/assets/stylesheets/bootstrap_migration.scss index d8e57834f9e..e24f8b1d4e8 100644 --- a/app/assets/stylesheets/bootstrap_migration.scss +++ b/app/assets/stylesheets/bootstrap_migration.scss @@ -36,6 +36,12 @@ html [type="button"], cursor: pointer; } +input[type="file"] { + // Bootstrap 4 file input height is taller by default + // which makes them look ugly + line-height: 1; +} + b, strong { font-weight: bold; -- cgit v1.2.1 From e8ecae7e0be12e6a1fe0e999d724ab5db9bb8b69 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 2 Jun 2018 11:55:42 +0900 Subject: Reveert build_relations and simply add a line for creating iid --- lib/gitlab/ci/pipeline/chain/populate.rb | 20 +++++++------------- spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb | 6 +++--- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 7a2a1c6a80b..f34c11ca3c2 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -8,7 +8,13 @@ module Gitlab PopulateError = Class.new(StandardError) def perform! - build_relations + # Allocate next IID. This operation must be outside of transactions of pipeline creations. + pipeline.ensure_project_iid! + + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) ## # Populate pipeline with all stages, and stages with builds. @@ -31,18 +37,6 @@ module Gitlab def break? pipeline.errors.any? end - - private - - def build_relations - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) - - # Allocate next IID. This operation must be outside of transactions of pipeline creations. - pipeline.ensure_project_iid! - end end end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index 7088233f237..e1766fc0ec9 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -140,10 +140,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do expect { step.perform! }.to raise_error(ActiveRecord::RecordNotSaved) end - it 'does not waste pipeline iid' do - step.perform rescue nil + it 'wastes pipeline iid' do + expect { step.perform! }.to raise_error - expect(InternalId.ci_pipelines.where(project_id: project.id).exists?).to be_falsy + expect(InternalId.ci_pipelines.where(project_id: project.id).exists?).to be_truthy end end end -- cgit v1.2.1 From 61df812ac688cb0848752f9f26f77d65eadf160a Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 2 Jun 2018 02:32:30 -0700 Subject: Fix attr_encryption key settings attr_encrypted does different things with `key` depending on what mode you are using: 1. In `:per_attribute_iv_and_salt` mode, it generates a hash with the salt: https://github.com/attr-encrypted/encryptor/blob/c3a62c4a9e74686dd95e0548f9dc2a361fdc95d1/lib/encryptor.rb#L77. There is no need to truncate the key to 32 bytes here. 2. In `:per_attribute_iv` mode, it sets the key directly to the password, so truncation to 32 bytes is necessary. Closes #47166 --- app/models/clusters/platforms/kubernetes.rb | 4 ++-- app/models/clusters/providers/gcp.rb | 2 +- .../unreleased/sh-fix-secrets-not-working.yml | 5 +++++ config/settings.rb | 23 ++++++++++++++-------- 4 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 changelogs/unreleased/sh-fix-secrets-not-working.yml diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 25eac5160f1..36631d57ad1 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -11,12 +11,12 @@ module Clusters attr_encrypted :password, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' attr_encrypted :token, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' before_validation :enforce_namespace_to_lower_case diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index eb2e42fd3fe..4db1bb35c12 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -11,7 +11,7 @@ module Clusters attr_encrypted :access_token, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' validates :gcp_project_id, diff --git a/changelogs/unreleased/sh-fix-secrets-not-working.yml b/changelogs/unreleased/sh-fix-secrets-not-working.yml new file mode 100644 index 00000000000..044a873ecd9 --- /dev/null +++ b/changelogs/unreleased/sh-fix-secrets-not-working.yml @@ -0,0 +1,5 @@ +--- +title: Fix attr_encryption key settings +merge_request: +author: +type: fixed diff --git a/config/settings.rb b/config/settings.rb index 58f38d103ea..3f3481bb65d 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -85,17 +85,24 @@ class Settings < Settingslogic File.expand_path(path, Rails.root) end - # Returns a 256-bit key for attr_encrypted - def attr_encrypted_db_key_base - # Ruby 2.4+ requires passing in the exact required length for OpenSSL keys - # (https://github.com/ruby/ruby/commit/ce635262f53b760284d56bb1027baebaaec175d1). - # Previous versions quietly truncated the input. - # - # The default mode for the attr_encrypted gem is to use a 256-bit key. - # We truncate the 128-byte string to 32 bytes. + # Ruby 2.4+ requires passing in the exact required length for OpenSSL keys + # (https://github.com/ruby/ruby/commit/ce635262f53b760284d56bb1027baebaaec175d1). + # Previous versions quietly truncated the input. + # + # Use this when using :per_attribute_iv mode for attr_encrypted. + # We have to truncate the string to 32 bytes for a 256-bit cipher. + def attr_encrypted_db_key_base_truncated Gitlab::Application.secrets.db_key_base[0..31] end + # This should be used for :per_attribute_salt_and_iv mode. There is no + # need to truncate the key because the encryptor will use the salt to + # generate a hash of the password: + # https://github.com/attr-encrypted/encryptor/blob/c3a62c4a9e74686dd95e0548f9dc2a361fdc95d1/lib/encryptor.rb#L77 + def attr_encrypted_db_key_base + Gitlab::Application.secrets.db_key_base + end + private def base_url(config) -- cgit v1.2.1 From a91999e4faddbb5f5fc8e38548f4544ba6a41adf Mon Sep 17 00:00:00 2001 From: Jasper Maes Date: Sat, 2 Jun 2018 01:52:15 +0200 Subject: Rails5 Fix arel from --- changelogs/unreleased/rails5-fix-46281.yml | 5 +++++ lib/gitlab/database/median.rb | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 changelogs/unreleased/rails5-fix-46281.yml diff --git a/changelogs/unreleased/rails5-fix-46281.yml b/changelogs/unreleased/rails5-fix-46281.yml new file mode 100644 index 00000000000..ee0b8531988 --- /dev/null +++ b/changelogs/unreleased/rails5-fix-46281.yml @@ -0,0 +1,5 @@ +--- +title: Rails5 fix arel from +merge_request: 19340 +author: Jasper Maes +type: fixed diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 74fed447289..3cac007a42c 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -143,8 +143,13 @@ module Gitlab .order(arel_table[column_sym]) ).as('row_id') - count = arel_table.from(arel_table.alias) - .project('COUNT(*)') + arel_from = if Gitlab.rails5? + arel_table.from.from(arel_table.alias) + else + arel_table.from(arel_table.alias) + end + + count = arel_from.project('COUNT(*)') .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) .as('ct') -- cgit v1.2.1 From ca466d5d152e1a4f35ef044f344d8b7b33617db2 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 2 Jun 2018 06:14:58 -0700 Subject: Fix missing key change in 20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb --- ...24104327_migrate_kubernetes_service_to_new_clusters_architectures.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb index 1586a7eb92f..a957f107405 100644 --- a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb +++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb @@ -48,7 +48,7 @@ class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migrati attr_encrypted :token, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' end -- cgit v1.2.1 From d65cdd441627cf73ad9e2554ca8d6912e851672d Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 2 Jun 2018 00:28:57 -0700 Subject: Fix intermittent failing spec in spec/support/helpers/cycle_analytics_helpers.rb There was a race condition in the spec where if the commit is created on disk within a second of the frozen `Timecop` time, the test fails. Closes #43981 --- spec/lib/gitlab/cycle_analytics/usage_data_spec.rb | 19 ++++++++--------- spec/support/helpers/cycle_analytics_helpers.rb | 24 +++++++++++----------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/spec/lib/gitlab/cycle_analytics/usage_data_spec.rb b/spec/lib/gitlab/cycle_analytics/usage_data_spec.rb index 56a316318cb..a785b17f682 100644 --- a/spec/lib/gitlab/cycle_analytics/usage_data_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/usage_data_spec.rb @@ -3,7 +3,12 @@ require 'spec_helper' describe Gitlab::CycleAnalytics::UsageData do describe '#to_json' do before do - Timecop.freeze do + # Since git commits only have second precision, round up to the + # nearest second to ensure we have accurate median and standard + # deviation calculations. + current_time = Time.at(Time.now.to_i) + + Timecop.freeze(current_time) do user = create(:user, :admin) projects = create_list(:project, 2, :repository) @@ -37,13 +42,7 @@ describe Gitlab::CycleAnalytics::UsageData do expected_values.each_pair do |op, value| expect(stage_values).to have_key(op) - - if op == :missing - expect(stage_values[op]).to eq(value) - else - # delta is used because of git timings that Timecop does not stub - expect(stage_values[op].to_i).to be_within(5).of(value.to_i) - end + expect(stage_values[op]).to eq(value) end end end @@ -58,8 +57,8 @@ describe Gitlab::CycleAnalytics::UsageData do missing: 0 }, plan: { - average: 2, - sd: 2, + average: 1, + sd: 0, missing: 0 }, code: { diff --git a/spec/support/helpers/cycle_analytics_helpers.rb b/spec/support/helpers/cycle_analytics_helpers.rb index 55359d36597..06a76d53354 100644 --- a/spec/support/helpers/cycle_analytics_helpers.rb +++ b/spec/support/helpers/cycle_analytics_helpers.rb @@ -4,12 +4,12 @@ module CycleAnalyticsHelpers create_commit("Commit for ##{issue.iid}", issue.project, user, branch_name) end - def create_commit(message, project, user, branch_name, count: 1) + def create_commit(message, project, user, branch_name, count: 1, commit_time: nil, skip_push_handler: false) repository = project.repository - oldrev = repository.commit(branch_name).sha + oldrev = repository.commit(branch_name)&.sha || Gitlab::Git::BLANK_SHA if Timecop.frozen? && Gitlab::GitalyClient.feature_enabled?(:operation_user_commit_files) - mock_gitaly_multi_action_dates(repository.raw) + mock_gitaly_multi_action_dates(repository.raw, commit_time) end commit_shas = Array.new(count) do |index| @@ -19,6 +19,8 @@ module CycleAnalyticsHelpers commit_sha end + return if skip_push_handler + GitPushService.new(project, user, oldrev: oldrev, @@ -44,13 +46,11 @@ module CycleAnalyticsHelpers project.repository.add_branch(user, source_branch, 'master') end - sha = project.repository.create_file( - user, - generate(:branch), - 'content', - message: commit_message, - branch_name: source_branch) - project.repository.commit(sha) + # Cycle analytic specs often test with frozen times, which causes metrics to be + # pinned to the current time. For example, in the plan stage, we assume that an issue + # milestone has been created before any code has been written. We add a second + # to ensure that the plan time is positive. + create_commit(commit_message, project, user, source_branch, commit_time: Time.now + 1.second, skip_push_handler: true) opts = { title: 'Awesome merge_request', @@ -116,9 +116,9 @@ module CycleAnalyticsHelpers protected: false) end - def mock_gitaly_multi_action_dates(raw_repository) + def mock_gitaly_multi_action_dates(raw_repository, commit_time) allow(raw_repository).to receive(:multi_action).and_wrap_original do |m, *args| - new_date = Time.now + new_date = commit_time || Time.now branch_update = m.call(*args) if branch_update.newrev -- cgit v1.2.1 From bd4bfcc6411e4819c0c67717095bb2e54e7bb6df Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 3 Jun 2018 03:31:41 -0700 Subject: Fix N+1 with source projects in merge requests API Now that we are checking `MergeRequest#for_fork?`, we also need the source project preloaded for a merge request. --- changelogs/unreleased/sh-fix-source-project-nplus-one.yml | 5 +++++ lib/api/merge_requests.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/sh-fix-source-project-nplus-one.yml diff --git a/changelogs/unreleased/sh-fix-source-project-nplus-one.yml b/changelogs/unreleased/sh-fix-source-project-nplus-one.yml new file mode 100644 index 00000000000..9d78ad6408c --- /dev/null +++ b/changelogs/unreleased/sh-fix-source-project-nplus-one.yml @@ -0,0 +1,5 @@ +--- +title: Fix N+1 with source_projects in merge requests API +merge_request: +author: +type: performance diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index b1e510d72de..278d53427f0 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -38,7 +38,7 @@ module API merge_requests = MergeRequestsFinder.new(current_user, args).execute .reorder(args[:order_by] => args[:sort]) merge_requests = paginate(merge_requests) - .preload(:target_project) + .preload(:source_project, :target_project) return merge_requests if args[:view] == 'simple' -- cgit v1.2.1 From e87031e4b127fe11044e2d761362f90fecda1537 Mon Sep 17 00:00:00 2001 From: Takuya Noguchi Date: Sat, 26 May 2018 23:21:20 +0900 Subject: Update email_spec to 2.2.0 --- Gemfile | 2 +- Gemfile.lock | 7 ++++--- changelogs/unreleased/46845-update-email_spec-to-2-2-0.yml | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 changelogs/unreleased/46845-update-email_spec-to-2-2-0.yml diff --git a/Gemfile b/Gemfile index 68c7b3dcb08..cd9cbcde359 100644 --- a/Gemfile +++ b/Gemfile @@ -374,7 +374,7 @@ end group :test do gem 'shoulda-matchers', '~> 3.1.2', require: false - gem 'email_spec', '~> 1.6.0' + gem 'email_spec', '~> 2.2.0' gem 'json-schema', '~> 2.8.0' gem 'webmock', '~> 2.3.2' gem 'rails-controller-testing' if rails5? # Rails5 only gem. diff --git a/Gemfile.lock b/Gemfile.lock index 2efd89bf40d..c2f11ca4c84 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -178,9 +178,10 @@ GEM dropzonejs-rails (0.7.2) rails (> 3.1) email_reply_trimmer (0.1.6) - email_spec (1.6.0) + email_spec (2.2.0) + htmlentities (~> 4.3.3) launchy (~> 2.1) - mail (~> 2.2) + mail (~> 2.7) encryptor (3.0.0) equalizer (0.0.11) erubis (2.7.0) @@ -1012,7 +1013,7 @@ DEPENDENCIES doorkeeper-openid_connect (~> 1.3) dropzonejs-rails (~> 0.7.1) email_reply_trimmer (~> 0.1) - email_spec (~> 1.6.0) + email_spec (~> 2.2.0) factory_bot_rails (~> 4.8.2) faraday (~> 0.12) fast_blank diff --git a/changelogs/unreleased/46845-update-email_spec-to-2-2-0.yml b/changelogs/unreleased/46845-update-email_spec-to-2-2-0.yml new file mode 100644 index 00000000000..bf501340769 --- /dev/null +++ b/changelogs/unreleased/46845-update-email_spec-to-2-2-0.yml @@ -0,0 +1,5 @@ +--- +title: Update email_spec to 2.2.0 +merge_request: 19164 +author: Takuya Noguchi +type: other -- cgit v1.2.1 From 3579ed016acbc3a512217d896a112747fcfd33bc Mon Sep 17 00:00:00 2001 From: Takuya Noguchi Date: Sun, 3 Jun 2018 22:17:36 +0900 Subject: Revert "Add a new have_html_escaped_body_text that match an HTML-escaped text" This reverts commit 517598ba10793efa02cb90379f78ab97c9c5b25d. --- spec/mailers/notify_spec.rb | 18 +++++++++--------- spec/support/matchers/email_matchers.rb | 5 ----- 2 files changed, 9 insertions(+), 14 deletions(-) delete mode 100644 spec/support/matchers/email_matchers.rb diff --git a/spec/mailers/notify_spec.rb b/spec/mailers/notify_spec.rb index 69eafbe4bbe..ecd3d309327 100644 --- a/spec/mailers/notify_spec.rb +++ b/spec/mailers/notify_spec.rb @@ -660,7 +660,7 @@ describe Notify do is_expected.to have_html_escaped_body_text project.full_name is_expected.to have_body_text project.web_url is_expected.to have_body_text project_member.invite_email - is_expected.to have_html_escaped_body_text invited_user.name + is_expected.to have_body_text invited_user.name end end @@ -932,7 +932,7 @@ describe Notify do end it 'contains the message from the note' do - is_expected.to have_html_escaped_body_text note.note + is_expected.to have_body_text note.note end it 'contains an introduction' do @@ -991,7 +991,7 @@ describe Notify do expect(to_emails).to eq([recipient.notification_email]) is_expected.to have_subject "Request to join the #{group.name} group" - is_expected.to have_html_escaped_body_text group.name + is_expected.to have_body_text group.name is_expected.to have_body_text group_group_members_url(group) is_expected.to have_body_text group_member.human_access end @@ -1010,7 +1010,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject "Access to the #{group.name} group was denied" - is_expected.to have_html_escaped_body_text group.name + is_expected.to have_body_text group.name is_expected.to have_body_text group.web_url end end @@ -1026,7 +1026,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject "Access to the #{group.name} group was granted" - is_expected.to have_html_escaped_body_text group.name + is_expected.to have_body_text group.name is_expected.to have_body_text group.web_url is_expected.to have_body_text group_member.human_access end @@ -1056,7 +1056,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject "Invitation to join the #{group.name} group" - is_expected.to have_html_escaped_body_text group.name + is_expected.to have_body_text group.name is_expected.to have_body_text group.web_url is_expected.to have_body_text group_member.human_access is_expected.to have_body_text group_member.invite_token @@ -1080,10 +1080,10 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject 'Invitation accepted' - is_expected.to have_html_escaped_body_text group.name + is_expected.to have_body_text group.name is_expected.to have_body_text group.web_url is_expected.to have_body_text group_member.invite_email - is_expected.to have_html_escaped_body_text invited_user.name + is_expected.to have_body_text invited_user.name end end @@ -1103,7 +1103,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject 'Invitation declined' - is_expected.to have_html_escaped_body_text group.name + is_expected.to have_body_text group.name is_expected.to have_body_text group.web_url is_expected.to have_body_text group_member.invite_email end diff --git a/spec/support/matchers/email_matchers.rb b/spec/support/matchers/email_matchers.rb deleted file mode 100644 index d9d59ec12ec..00000000000 --- a/spec/support/matchers/email_matchers.rb +++ /dev/null @@ -1,5 +0,0 @@ -RSpec::Matchers.define :have_html_escaped_body_text do |expected| - match do |actual| - expect(actual).to have_body_text(ERB::Util.html_escape(expected)) - end -end -- cgit v1.2.1 From d46c12290d74e9faac888bf5de5cc1d84a00b675 Mon Sep 17 00:00:00 2001 From: Takuya Noguchi Date: Sun, 3 Jun 2018 22:22:50 +0900 Subject: Replace have_html_espaced_body_text after 517598ba --- spec/mailers/notify_spec.rb | 38 +++++++++++----------- .../shared_examples/notify_shared_examples.rb | 4 +-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/spec/mailers/notify_spec.rb b/spec/mailers/notify_spec.rb index ecd3d309327..775ca4ba0eb 100644 --- a/spec/mailers/notify_spec.rb +++ b/spec/mailers/notify_spec.rb @@ -68,7 +68,7 @@ describe Notify do end it 'contains the description' do - is_expected.to have_html_escaped_body_text issue.description + is_expected.to have_body_text issue.description end it 'does not add a reason header' do @@ -89,7 +89,7 @@ describe Notify do end it 'contains a link to note author' do - is_expected.to have_html_escaped_body_text(issue.author_name) + is_expected.to have_body_text(issue.author_name) is_expected.to have_body_text 'created an issue:' end end @@ -115,8 +115,8 @@ describe Notify do it 'has the correct subject and body' do aggregate_failures do is_expected.to have_referable_subject(issue, reply: true) - is_expected.to have_html_escaped_body_text(previous_assignee.name) - is_expected.to have_html_escaped_body_text(assignee.name) + is_expected.to have_body_text(previous_assignee.name) + is_expected.to have_body_text(assignee.name) is_expected.to have_body_text(project_issue_path(project, issue)) end end @@ -190,7 +190,7 @@ describe Notify do aggregate_failures do is_expected.to have_referable_subject(issue, reply: true) is_expected.to have_body_text(status) - is_expected.to have_html_escaped_body_text(current_user.name) + is_expected.to have_body_text(current_user.name) is_expected.to have_body_text(project_issue_path project, issue) end end @@ -243,7 +243,7 @@ describe Notify do end it 'contains the description' do - is_expected.to have_html_escaped_body_text merge_request.description + is_expected.to have_body_text merge_request.description end context 'when sent with a reason' do @@ -260,7 +260,7 @@ describe Notify do end it 'contains a link to note author' do - is_expected.to have_html_escaped_body_text merge_request.author_name + is_expected.to have_body_text merge_request.author_name is_expected.to have_body_text 'created a merge request:' end end @@ -286,9 +286,9 @@ describe Notify do it 'has the correct subject and body' do aggregate_failures do is_expected.to have_referable_subject(merge_request, reply: true) - is_expected.to have_html_escaped_body_text(previous_assignee.name) + is_expected.to have_body_text(previous_assignee.name) is_expected.to have_body_text(project_merge_request_path(project, merge_request)) - is_expected.to have_html_escaped_body_text(assignee.name) + is_expected.to have_body_text(assignee.name) end end @@ -358,7 +358,7 @@ describe Notify do aggregate_failures do is_expected.to have_referable_subject(merge_request, reply: true) is_expected.to have_body_text(status) - is_expected.to have_html_escaped_body_text(current_user.name) + is_expected.to have_body_text(current_user.name) is_expected.to have_body_text(project_merge_request_path(project, merge_request)) end end @@ -526,7 +526,7 @@ describe Notify do it 'has the correct subject and body' do is_expected.to have_referable_subject(project_snippet, reply: true) - is_expected.to have_html_escaped_body_text project_snippet_note.note + is_expected.to have_body_text project_snippet_note.note end end @@ -539,7 +539,7 @@ describe Notify do it 'has the correct subject and body' do is_expected.to have_subject("#{project.name} | Project was moved") - is_expected.to have_html_escaped_body_text project.full_name + is_expected.to have_body_text project.full_name is_expected.to have_body_text(project.ssh_url_to_repo) end end @@ -566,7 +566,7 @@ describe Notify do expect(to_emails).to eq([recipient.notification_email]) is_expected.to have_subject "Request to join the #{project.full_name} project" - is_expected.to have_html_escaped_body_text project.full_name + is_expected.to have_body_text project.full_name is_expected.to have_body_text project_project_members_url(project) is_expected.to have_body_text project_member.human_access end @@ -586,7 +586,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject "Access to the #{project.full_name} project was denied" - is_expected.to have_html_escaped_body_text project.full_name + is_expected.to have_body_text project.full_name is_expected.to have_body_text project.web_url end end @@ -603,7 +603,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject "Access to the #{project.full_name} project was granted" - is_expected.to have_html_escaped_body_text project.full_name + is_expected.to have_body_text project.full_name is_expected.to have_body_text project.web_url is_expected.to have_body_text project_member.human_access end @@ -633,7 +633,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject "Invitation to join the #{project.full_name} project" - is_expected.to have_html_escaped_body_text project.full_name + is_expected.to have_body_text project.full_name is_expected.to have_body_text project.full_name is_expected.to have_body_text project_member.human_access is_expected.to have_body_text project_member.invite_token @@ -657,7 +657,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject 'Invitation accepted' - is_expected.to have_html_escaped_body_text project.full_name + is_expected.to have_body_text project.full_name is_expected.to have_body_text project.web_url is_expected.to have_body_text project_member.invite_email is_expected.to have_body_text invited_user.name @@ -680,7 +680,7 @@ describe Notify do it 'contains all the useful information' do is_expected.to have_subject 'Invitation declined' - is_expected.to have_html_escaped_body_text project.full_name + is_expected.to have_body_text project.full_name is_expected.to have_body_text project.web_url is_expected.to have_body_text project_member.invite_email end @@ -1396,7 +1396,7 @@ describe Notify do it 'has the correct subject and body' do is_expected.to have_referable_subject(personal_snippet, reply: true) - is_expected.to have_html_escaped_body_text personal_snippet_note.note + is_expected.to have_body_text personal_snippet_note.note end end end diff --git a/spec/support/shared_examples/notify_shared_examples.rb b/spec/support/shared_examples/notify_shared_examples.rb index 43fdaddf545..d176d3fa425 100644 --- a/spec/support/shared_examples/notify_shared_examples.rb +++ b/spec/support/shared_examples/notify_shared_examples.rb @@ -212,7 +212,7 @@ shared_examples 'a note email' do end it 'contains the message from the note' do - is_expected.to have_html_escaped_body_text note.note + is_expected.to have_body_text note.note end it 'does not contain note author' do @@ -225,7 +225,7 @@ shared_examples 'a note email' do end it 'contains a link to note author' do - is_expected.to have_html_escaped_body_text note.author_name + is_expected.to have_body_text note.author_name end end end -- cgit v1.2.1 From 0b648a492060654126de86d813c1b877535de832 Mon Sep 17 00:00:00 2001 From: Takuya Noguchi Date: Mon, 4 Jun 2018 00:15:58 +0900 Subject: Update selenium-webdriver to 3.12.0 --- Gemfile | 2 +- Gemfile.lock | 8 ++++---- .../unreleased/47183-update-selenium-webdriver-to-3-12-0.yml | 5 +++++ 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 changelogs/unreleased/47183-update-selenium-webdriver-to-3-12-0.yml diff --git a/Gemfile b/Gemfile index 68c7b3dcb08..e3e7ef2aa27 100644 --- a/Gemfile +++ b/Gemfile @@ -342,7 +342,7 @@ group :development, :test do gem 'capybara', '~> 2.15' gem 'capybara-screenshot', '~> 1.0.0' - gem 'selenium-webdriver', '~> 3.5' + gem 'selenium-webdriver', '~> 3.12' gem 'spring', '~> 2.0.0' gem 'spring-commands-rspec', '~> 1.0.4' diff --git a/Gemfile.lock b/Gemfile.lock index 2efd89bf40d..767472a8a7a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -115,7 +115,7 @@ GEM mime-types (>= 1.16) cause (0.1) charlock_holmes (0.7.6) - childprocess (0.7.0) + childprocess (0.9.0) ffi (~> 1.0, >= 1.0.11) chronic (0.10.2) chronic_duration (0.10.6) @@ -828,9 +828,9 @@ GEM activesupport (>= 3.1) select2-rails (3.5.9.3) thor (~> 0.14) - selenium-webdriver (3.5.0) + selenium-webdriver (3.12.0) childprocess (~> 0.5) - rubyzip (~> 1.0) + rubyzip (~> 1.2) sentry-raven (2.7.2) faraday (>= 0.7.6, < 1.0) settingslogic (2.0.9) @@ -1154,7 +1154,7 @@ DEPENDENCIES scss_lint (~> 0.56.0) seed-fu (~> 2.3.7) select2-rails (~> 3.5.9) - selenium-webdriver (~> 3.5) + selenium-webdriver (~> 3.12) sentry-raven (~> 2.7) settingslogic (~> 2.0.9) sham_rack (~> 1.3.6) diff --git a/changelogs/unreleased/47183-update-selenium-webdriver-to-3-12-0.yml b/changelogs/unreleased/47183-update-selenium-webdriver-to-3-12-0.yml new file mode 100644 index 00000000000..b0d51d810f2 --- /dev/null +++ b/changelogs/unreleased/47183-update-selenium-webdriver-to-3-12-0.yml @@ -0,0 +1,5 @@ +--- +title: Update selenium-webdriver to 3.12.0 +merge_request: 19351 +author: Takuya Noguchi +type: other -- cgit v1.2.1 From b0ec77663254f4a0c8abccd7ee9fdde23a55fb27 Mon Sep 17 00:00:00 2001 From: Ash McKenzie Date: Mon, 4 Jun 2018 12:23:18 +1000 Subject: Bump octokit to 4.9 --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index e3e7ef2aa27..ae1301c335a 100644 --- a/Gemfile +++ b/Gemfile @@ -384,7 +384,7 @@ group :test do gem 'test-prof', '~> 0.2.5' end -gem 'octokit', '~> 4.8' +gem 'octokit', '~> 4.9' gem 'mail_room', '~> 0.9.1' diff --git a/Gemfile.lock b/Gemfile.lock index 767472a8a7a..e6e8f3d11bc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -517,7 +517,7 @@ GEM multi_json (~> 1.3) multi_xml (~> 0.5) rack (>= 1.2, < 3) - octokit (4.8.0) + octokit (4.9.0) sawyer (~> 0.8.0, >= 0.5.3) omniauth (1.8.1) hashie (>= 3.4.6, < 3.6.0) @@ -1084,7 +1084,7 @@ DEPENDENCIES net-ssh (~> 4.2.0) nokogiri (~> 1.8.2) oauth2 (~> 1.4) - octokit (~> 4.8) + octokit (~> 4.9) omniauth (~> 1.8) omniauth-auth0 (~> 2.0.0) omniauth-authentiq (~> 0.3.3) -- cgit v1.2.1 From eb05d475b7e82b943f5a72b8adf41b9bce519382 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 4 Jun 2018 12:12:02 +0900 Subject: Fix wording in spec. Add PIPELINE_IID in examples of debugged variables in documants. --- doc/ci/variables/README.md | 3 +++ spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index dfea10314b9..aa4395b01a9 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -353,6 +353,8 @@ Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-mach ++ CI_PROJECT_URL=https://example.com/gitlab-examples/ci-debug-trace ++ export CI_PIPELINE_ID=52666 ++ CI_PIPELINE_ID=52666 +++ export CI_PIPELINE_IID=123 +++ CI_PIPELINE_IID=123 ++ export CI_RUNNER_ID=1337 ++ CI_RUNNER_ID=1337 ++ export CI_RUNNER_DESCRIPTION=shared-runners-manager-1.example.com @@ -440,6 +442,7 @@ export CI_JOB_MANUAL="true" export CI_JOB_TRIGGERED="true" export CI_JOB_TOKEN="abcde-1234ABCD5678ef" export CI_PIPELINE_ID="1000" +export CI_PIPELINE_IID="10" export CI_PROJECT_ID="34" export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce" export CI_PROJECT_NAME="gitlab-ce" diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb index e1766fc0ec9..c5a4d9b4778 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -143,7 +143,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do it 'wastes pipeline iid' do expect { step.perform! }.to raise_error - expect(InternalId.ci_pipelines.where(project_id: project.id).exists?).to be_truthy + expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0 end end end @@ -157,7 +157,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do end context 'when variables policy is specified' do - shared_examples_for 'populates pipeline according to used policies' do + shared_examples_for 'a correct pipeline' do it 'populates pipeline according to used policies' do step.perform! @@ -177,7 +177,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do build(:ci_pipeline, ref: 'master', config: config) end - it_behaves_like 'populates pipeline according to used policies' + it_behaves_like 'a correct pipeline' context 'when variables expression is specified' do context 'when pipeline iid is the subject' do @@ -186,7 +186,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do prod: { script: 'cap prod', only: { variables: ["$CI_PIPELINE_IID == '1000'"] } } } end - it_behaves_like 'populates pipeline according to used policies' + it_behaves_like 'a correct pipeline' end end end -- cgit v1.2.1 From 4022466e25cdfb1c320b425bb9bf810ffff1e417 Mon Sep 17 00:00:00 2001 From: Natho Date: Mon, 4 Jun 2018 14:08:35 +0930 Subject: Update IPs to valid example IPs. As per: https://tools.ietf.org/html/rfc5737 --- doc/administration/pages/index.md | 22 +++++++++++----------- doc/administration/pages/source.md | 18 +++++++++--------- doc/install/kubernetes/gitlab_omnibus.md | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index c0221533f13..9b1297ca4ba 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -83,12 +83,12 @@ you need to add a [wildcard DNS A record][wiki-wildcard-dns] pointing to the host that GitLab runs. For example, an entry would look like this: ``` -*.example.io. 1800 IN A 1.1.1.1 +*.example.io. 1800 IN A 192.0.2.1 *.example.io. 1800 IN AAAA 2001::1 ``` where `example.io` is the domain under which GitLab Pages will be served -and `1.1.1.1` is the IPv4 address of your GitLab instance and `2001::1` is the +and `192.0.2.1` is the IPv4 address of your GitLab instance and `2001::1` is the IPv6 address. If you don't have IPv6, you can omit the AAAA record. > **Note:** @@ -193,13 +193,13 @@ world. Custom domains are supported, but no TLS. ```shell pages_external_url "http://example.io" - nginx['listen_addresses'] = ['1.1.1.1'] + nginx['listen_addresses'] = ['192.0.2.1'] pages_nginx['enable'] = false - gitlab_pages['external_http'] = ['1.1.1.2:80', '[2001::2]:80'] + gitlab_pages['external_http'] = ['192.0.2.2:80', '[2001::2]:80'] ``` - where `1.1.1.1` is the primary IP address that GitLab is listening to and - `1.1.1.2` and `2001::2` are the secondary IPs the GitLab Pages daemon + where `192.0.2.1` is the primary IP address that GitLab is listening to and + `192.0.2.2` and `2001::2` are the secondary IPs the GitLab Pages daemon listens on. If you don't have IPv6, you can omit the IPv6 address. 1. [Reconfigure GitLab][reconfigure] @@ -228,16 +228,16 @@ world. Custom domains and TLS are supported. ```shell pages_external_url "https://example.io" - nginx['listen_addresses'] = ['1.1.1.1'] + nginx['listen_addresses'] = ['192.0.2.1'] pages_nginx['enable'] = false gitlab_pages['cert'] = "/etc/gitlab/ssl/example.io.crt" gitlab_pages['cert_key'] = "/etc/gitlab/ssl/example.io.key" - gitlab_pages['external_http'] = ['1.1.1.2:80', '[2001::2]:80'] - gitlab_pages['external_https'] = ['1.1.1.2:443', '[2001::2]:443'] + gitlab_pages['external_http'] = ['192.0.2.2:80', '[2001::2]:80'] + gitlab_pages['external_https'] = ['192.0.2.2:443', '[2001::2]:443'] ``` - where `1.1.1.1` is the primary IP address that GitLab is listening to and - `1.1.1.2` and `2001::2` are the secondary IPs where the GitLab Pages daemon + where `192.0.2.1` is the primary IP address that GitLab is listening to and + `192.0.2.2` and `2001::2` are the secondary IPs where the GitLab Pages daemon listens on. If you don't have IPv6, you can omit the IPv6 address. 1. [Reconfigure GitLab][reconfigure] diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index a45c3306457..4e40a7cb18d 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -67,11 +67,11 @@ you need to add a [wildcard DNS A record][wiki-wildcard-dns] pointing to the host that GitLab runs. For example, an entry would look like this: ``` -*.example.io. 1800 IN A 1.1.1.1 +*.example.io. 1800 IN A 192.0.2.1 ``` where `example.io` is the domain under which GitLab Pages will be served -and `1.1.1.1` is the IP address of your GitLab instance. +and `192.0.2.1` is the IP address of your GitLab instance. > **Note:** You should not use the GitLab domain to serve user pages. For more information @@ -253,7 +253,7 @@ world. Custom domains are supported, but no TLS. port: 80 https: false - external_http: 1.1.1.2:80 + external_http: 192.0.2.2:80 ``` 1. Edit `/etc/default/gitlab` and set `gitlab_pages_enabled` to `true` in @@ -263,7 +263,7 @@ world. Custom domains are supported, but no TLS. ``` gitlab_pages_enabled=true - gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -listen-http 1.1.1.2:80" + gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -listen-http 192.0.2.2:80" ``` 1. Copy the `gitlab-pages-ssl` Nginx configuration file: @@ -274,7 +274,7 @@ world. Custom domains are supported, but no TLS. ``` 1. Edit all GitLab related configs in `/etc/nginx/site-available/` and replace - `0.0.0.0` with `1.1.1.1`, where `1.1.1.1` the primary IP where GitLab + `0.0.0.0` with `192.0.2.1`, where `192.0.2.1` the primary IP where GitLab listens to. 1. Restart NGINX 1. [Restart GitLab][restart] @@ -320,8 +320,8 @@ world. Custom domains and TLS are supported. port: 443 https: true - external_http: 1.1.1.2:80 - external_https: 1.1.1.2:443 + external_http: 192.0.2.2:80 + external_https: 192.0.2.2:443 ``` 1. Edit `/etc/default/gitlab` and set `gitlab_pages_enabled` to `true` in @@ -333,7 +333,7 @@ world. Custom domains and TLS are supported. ``` gitlab_pages_enabled=true - gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -listen-http 1.1.1.2:80 -listen-https 1.1.1.2:443 -root-cert /path/to/example.io.crt -root-key /path/to/example.io.key + gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -listen-http 192.0.2.2:80 -listen-https 192.0.2.2:443 -root-cert /path/to/example.io.crt -root-key /path/to/example.io.key ``` 1. Copy the `gitlab-pages-ssl` Nginx configuration file: @@ -344,7 +344,7 @@ world. Custom domains and TLS are supported. ``` 1. Edit all GitLab related configs in `/etc/nginx/site-available/` and replace - `0.0.0.0` with `1.1.1.1`, where `1.1.1.1` the primary IP where GitLab + `0.0.0.0` with `192.0.2.1`, where `192.0.2.1` the primary IP where GitLab listens to. 1. Restart NGINX 1. [Restart GitLab][restart] diff --git a/doc/install/kubernetes/gitlab_omnibus.md b/doc/install/kubernetes/gitlab_omnibus.md index 98af87455ec..e1d1969651e 100644 --- a/doc/install/kubernetes/gitlab_omnibus.md +++ b/doc/install/kubernetes/gitlab_omnibus.md @@ -144,7 +144,7 @@ helm install --name gitlab -f values.yaml gitlab/gitlab-omnibus or passing them on the command line: ```bash -helm install --name gitlab --set baseDomain=gitlab.io,baseIP=1.1.1.1,gitlab=ee,gitlabEELicense=$LICENSE,legoEmail=email@gitlab.com gitlab/gitlab-omnibus +helm install --name gitlab --set baseDomain=gitlab.io,baseIP=192.0.2.1,gitlab=ee,gitlabEELicense=$LICENSE,legoEmail=email@gitlab.com gitlab/gitlab-omnibus ``` ## Updating GitLab using the Helm Chart -- cgit v1.2.1 From 89b4304f12cd37d8715c274cdee080e95f2d3bad Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 29 May 2018 17:06:14 +0900 Subject: Add background migrations to arhive legacy traces --- .../20180529152628_archive_legacy_traces.rb | 44 ++++++++++++ db/schema.rb | 2 +- .../background_migration/archive_legacy_traces.rb | 79 ++++++++++++++++++++++ .../archive_legacy_traces_spec.rb | 60 ++++++++++++++++ spec/migrations/archive_legacy_traces_spec.rb | 45 ++++++++++++ 5 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 db/post_migrate/20180529152628_archive_legacy_traces.rb create mode 100644 lib/gitlab/background_migration/archive_legacy_traces.rb create mode 100644 spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb create mode 100644 spec/migrations/archive_legacy_traces_spec.rb diff --git a/db/post_migrate/20180529152628_archive_legacy_traces.rb b/db/post_migrate/20180529152628_archive_legacy_traces.rb new file mode 100644 index 00000000000..78ec1ab1a94 --- /dev/null +++ b/db/post_migrate/20180529152628_archive_legacy_traces.rb @@ -0,0 +1,44 @@ +class ArchiveLegacyTraces < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 10_000 + BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces' + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled # Disable STI + + scope :finished, -> { where(status: [:success, :failed, :canceled]) } + + scope :without_new_traces, ->() do + where('NOT EXISTS (?)', + ::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + end + end + + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + + enum file_type: { + archive: 1, + metadata: 2, + trace: 3 + } + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + ::ArchiveLegacyTraces::Build.finished.without_new_traces, + BACKGROUND_MIGRATION_CLASS, + 5.minutes, + batch_size: BATCH_SIZE) + end + + def down + # noop + end +end diff --git a/db/schema.rb b/db/schema.rb index 97247387bc7..a8f8e14a3fc 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180529093006) do +ActiveRecord::Schema.define(version: 20180529152628) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb new file mode 100644 index 00000000000..9741a7c181e --- /dev/null +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class ArchiveLegacyTraces + class Build < ActiveRecord::Base + include ::HasStatus + + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled # Disable STI + + belongs_to :project, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Project' + has_one :job_artifacts_trace, -> () { where(file_type: ArchiveLegacyTraces::JobArtifact.file_types[:trace]) }, class_name: 'ArchiveLegacyTraces::JobArtifact', foreign_key: :job_id + has_many :trace_chunks, foreign_key: :build_id, class_name: 'ArchiveLegacyTraces::BuildTraceChunk' + + scope :finished, -> { where(status: [:success, :failed, :canceled]) } + + scope :without_new_traces, ->() do + finished.where('NOT EXISTS (?)', + BackgroundMigration::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + end + + def trace + ::Gitlab::Ci::Trace.new(self) + end + + def trace=(data) + raise NotImplementedError + end + + def old_trace + read_attribute(:trace) + end + + def erase_old_trace! + update_column(:trace, nil) + end + end + + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + + belongs_to :build + belongs_to :project + + mount_uploader :file, JobArtifactUploader + + enum file_type: { + archive: 1, + metadata: 2, + trace: 3 + } + end + + class BuildTraceChunk < ActiveRecord::Base + self.table_name = 'ci_build_trace_chunks' + + belongs_to :build + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + has_many :builds, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Build' + end + + def perform(start_id, stop_id) + BackgroundMigration::ArchiveLegacyTraces::Build + .finished + .without_new_traces + .where(id: (start_id..stop_id)).find_each do |build| + build.trace.archive! + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb new file mode 100644 index 00000000000..ecc6eea9284 --- /dev/null +++ b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb @@ -0,0 +1,60 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 20180529152628 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:builds) { table(:ci_builds) } + let(:job_artifacts) { table(:ci_job_artifacts) } + + before do + namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) + build = builds.create!(id: 1, project_id: 123, status: 'success') + + @legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, + build.created_at.utc.strftime("%Y_%m"), + build.project_id.to_s) + + FileUtils.mkdir_p(@legacy_trace_dir) + + @legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log") + end + + context 'when trace file exsits at the right place' do + before do + File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') } + end + + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(File.exist?(@legacy_trace_path)).to be_truthy + + described_class.new.perform(1, 1) + + expect(job_artifacts.count).to eq(1) + expect(File.exist?(@legacy_trace_path)).to be_falsy + expect(File.read(new_trace_path)).to eq('aiueo') + end + end + + context 'when trace file does not exsits at the right place' do + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(File.exist?(@legacy_trace_path)).to be_falsy + + described_class.new.perform(1, 1) + + expect(job_artifacts.count).to eq(0) + end + end + + def new_trace_path + job_artifact = job_artifacts.first + + disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) + creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') + + File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash, + creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log') + end +end diff --git a/spec/migrations/archive_legacy_traces_spec.rb b/spec/migrations/archive_legacy_traces_spec.rb new file mode 100644 index 00000000000..fc61c4bec17 --- /dev/null +++ b/spec/migrations/archive_legacy_traces_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180529152628_archive_legacy_traces') + +describe ArchiveLegacyTraces, :migration do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:builds) { table(:ci_builds) } + let(:job_artifacts) { table(:ci_job_artifacts) } + + before do + namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) + build = builds.create!(id: 1) + + @legacy_trace_path = File.join( + Settings.gitlab_ci.builds_path, + build.created_at.utc.strftime("%Y_%m"), + build.project_id.to_s, + "#{job.id}.log" + ) + + File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') } + end + + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(File.exist?(@legacy_trace_path)).to be_truthy + + migrate! + + expect(job_artifacts.count).to eq(1) + expect(File.exist?(@legacy_trace_path)).to be_falsy + expect(File.exist?(new_trace_path)).to be_truthy + end + + def new_trace_path + job_artifact = job_artifacts.first + + disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) + creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') + + File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, + creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s) + end +end -- cgit v1.2.1 From b626fcc045dda7ac25b1b7d01229589d8fd00521 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Jun 2018 12:08:54 +0900 Subject: Add changelog --- .../unreleased/add-background-migrations-for-not-archived-traces.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/add-background-migrations-for-not-archived-traces.yml diff --git a/changelogs/unreleased/add-background-migrations-for-not-archived-traces.yml b/changelogs/unreleased/add-background-migrations-for-not-archived-traces.yml new file mode 100644 index 00000000000..b1b23c477df --- /dev/null +++ b/changelogs/unreleased/add-background-migrations-for-not-archived-traces.yml @@ -0,0 +1,5 @@ +--- +title: Add background migrations for archiving legacy job traces +merge_request: 19194 +author: +type: performance -- cgit v1.2.1 From 0d00d02e842a0c4b22d213e00143a08d97597000 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Jun 2018 14:21:50 +0900 Subject: Directly refer application code from migration code --- app/models/ci/build.rb | 5 ++ .../20180529152628_archive_legacy_traces.rb | 19 ++---- .../background_migration/archive_legacy_traces.rb | 77 ++++------------------ lib/tasks/gitlab/traces.rake | 4 +- .../archive_legacy_traces_spec.rb | 2 +- 5 files changed, 22 insertions(+), 85 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 75fd55a8f7b..6cbf9108244 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -55,6 +55,11 @@ module Ci where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive) end + + scope :without_archived_trace, ->() do + where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace) + end + scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } diff --git a/db/post_migrate/20180529152628_archive_legacy_traces.rb b/db/post_migrate/20180529152628_archive_legacy_traces.rb index 78ec1ab1a94..12f219f606c 100644 --- a/db/post_migrate/20180529152628_archive_legacy_traces.rb +++ b/db/post_migrate/20180529152628_archive_legacy_traces.rb @@ -2,7 +2,7 @@ class ArchiveLegacyTraces < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false - BATCH_SIZE = 10_000 + BATCH_SIZE = 5000 BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces' disable_ddl_transaction! @@ -14,25 +14,14 @@ class ArchiveLegacyTraces < ActiveRecord::Migration scope :finished, -> { where(status: [:success, :failed, :canceled]) } - scope :without_new_traces, ->() do - where('NOT EXISTS (?)', - ::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + scope :without_archived_trace, -> do + where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)') end end - class JobArtifact < ActiveRecord::Base - self.table_name = 'ci_job_artifacts' - - enum file_type: { - archive: 1, - metadata: 2, - trace: 3 - } - end - def up queue_background_migration_jobs_by_range_at_intervals( - ::ArchiveLegacyTraces::Build.finished.without_new_traces, + ::ArchiveLegacyTraces::Build.finished.without_archived_trace, BACKGROUND_MIGRATION_CLASS, 5.minutes, batch_size: BATCH_SIZE) diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 9741a7c181e..0999ea95e56 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -5,73 +5,18 @@ module Gitlab module BackgroundMigration class ArchiveLegacyTraces - class Build < ActiveRecord::Base - include ::HasStatus - - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled # Disable STI - - belongs_to :project, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Project' - has_one :job_artifacts_trace, -> () { where(file_type: ArchiveLegacyTraces::JobArtifact.file_types[:trace]) }, class_name: 'ArchiveLegacyTraces::JobArtifact', foreign_key: :job_id - has_many :trace_chunks, foreign_key: :build_id, class_name: 'ArchiveLegacyTraces::BuildTraceChunk' - - scope :finished, -> { where(status: [:success, :failed, :canceled]) } - - scope :without_new_traces, ->() do - finished.where('NOT EXISTS (?)', - BackgroundMigration::ArchiveLegacyTraces::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) - end - - def trace - ::Gitlab::Ci::Trace.new(self) - end - - def trace=(data) - raise NotImplementedError - end - - def old_trace - read_attribute(:trace) - end - - def erase_old_trace! - update_column(:trace, nil) - end - end - - class JobArtifact < ActiveRecord::Base - self.table_name = 'ci_job_artifacts' - - belongs_to :build - belongs_to :project - - mount_uploader :file, JobArtifactUploader - - enum file_type: { - archive: 1, - metadata: 2, - trace: 3 - } - end - - class BuildTraceChunk < ActiveRecord::Base - self.table_name = 'ci_build_trace_chunks' - - belongs_to :build - end - - class Project < ActiveRecord::Base - self.table_name = 'projects' - - has_many :builds, foreign_key: :project_id, class_name: 'ArchiveLegacyTraces::Build' - end - def perform(start_id, stop_id) - BackgroundMigration::ArchiveLegacyTraces::Build - .finished - .without_new_traces - .where(id: (start_id..stop_id)).find_each do |build| - build.trace.archive! + # This background migrations directly refer ::Ci::Build model which is defined in application code. + # In general, migration code should be isolated as much as possible in order to be idempotent. + # However, `archive!` logic is too complicated to be replicated. So we chose a way to refer ::Ci::Build directly + # and we don't change the `archive!` logic until 11.1 + ::Ci::Build.finished.without_archived_trace + .where(id: start_id..stop_id).find_each do |build| + begin + build.trace.archive! + rescue => e + Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" + end end end end diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake index fd2a4f2d11a..ddcca69711f 100644 --- a/lib/tasks/gitlab/traces.rake +++ b/lib/tasks/gitlab/traces.rake @@ -8,9 +8,7 @@ namespace :gitlab do logger = Logger.new(STDOUT) logger.info('Archiving legacy traces') - Ci::Build.finished - .where('NOT EXISTS (?)', - Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + Ci::Build.finished.without_archived_trace .order(id: :asc) .find_in_batches(batch_size: 1000) do |jobs| job_ids = jobs.map { |job| [job.id] } diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb index ecc6eea9284..1a62b30ce81 100644 --- a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb +++ b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb @@ -14,7 +14,7 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2 @legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, build.created_at.utc.strftime("%Y_%m"), build.project_id.to_s) - + FileUtils.mkdir_p(@legacy_trace_dir) @legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log") -- cgit v1.2.1 From bcd664f53a4009bc752fbc47e1c4d6f76c0b8cc2 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Jun 2018 15:04:47 +0900 Subject: Fix specs. Rename migration file name which was conflicted with background migration's. --- .../20180529152628_archive_legacy_traces.rb | 33 ---------------- ...0529152628_schedule_to_archive_legacy_traces.rb | 33 ++++++++++++++++ .../archive_legacy_traces_spec.rb | 35 ++++------------- spec/migrations/archive_legacy_traces_spec.rb | 45 ---------------------- .../schedule_to_archive_legacy_traces_spec.rb | 45 ++++++++++++++++++++++ spec/support/trace/trace_helpers.rb | 23 +++++++++++ 6 files changed, 109 insertions(+), 105 deletions(-) delete mode 100644 db/post_migrate/20180529152628_archive_legacy_traces.rb create mode 100644 db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb delete mode 100644 spec/migrations/archive_legacy_traces_spec.rb create mode 100644 spec/migrations/schedule_to_archive_legacy_traces_spec.rb create mode 100644 spec/support/trace/trace_helpers.rb diff --git a/db/post_migrate/20180529152628_archive_legacy_traces.rb b/db/post_migrate/20180529152628_archive_legacy_traces.rb deleted file mode 100644 index 12f219f606c..00000000000 --- a/db/post_migrate/20180529152628_archive_legacy_traces.rb +++ /dev/null @@ -1,33 +0,0 @@ -class ArchiveLegacyTraces < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - BATCH_SIZE = 5000 - BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces' - - disable_ddl_transaction! - - class Build < ActiveRecord::Base - include EachBatch - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled # Disable STI - - scope :finished, -> { where(status: [:success, :failed, :canceled]) } - - scope :without_archived_trace, -> do - where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)') - end - end - - def up - queue_background_migration_jobs_by_range_at_intervals( - ::ArchiveLegacyTraces::Build.finished.without_archived_trace, - BACKGROUND_MIGRATION_CLASS, - 5.minutes, - batch_size: BATCH_SIZE) - end - - def down - # noop - end -end diff --git a/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb b/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb new file mode 100644 index 00000000000..ea782e1596b --- /dev/null +++ b/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb @@ -0,0 +1,33 @@ +class ScheduleToArchiveLegacyTraces < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 5000 + BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces' + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled # Disable STI + + scope :finished, -> { where(status: [:success, :failed, :canceled]) } + + scope :without_archived_trace, -> do + where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)') + end + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + ::ScheduleToArchiveLegacyTraces::Build.finished.without_archived_trace, + BACKGROUND_MIGRATION_CLASS, + 5.minutes, + batch_size: BATCH_SIZE) + end + + def down + # noop + end +end diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb index 1a62b30ce81..0765f4149f9 100644 --- a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb +++ b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 20180529152628 do + include TraceHelpers + let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } let(:builds) { table(:ci_builds) } @@ -9,52 +11,31 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2 before do namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) - build = builds.create!(id: 1, project_id: 123, status: 'success') - - @legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, - build.created_at.utc.strftime("%Y_%m"), - build.project_id.to_s) - - FileUtils.mkdir_p(@legacy_trace_dir) - - @legacy_trace_path = File.join(@legacy_trace_dir, "#{build.id}.log") + @build = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build') end context 'when trace file exsits at the right place' do before do - File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') } + create_legacy_trace(@build, 'aiueo') end it 'correctly archive legacy traces' do expect(job_artifacts.count).to eq(0) - expect(File.exist?(@legacy_trace_path)).to be_truthy + expect(File.exist?(legacy_trace_path(@build))).to be_truthy described_class.new.perform(1, 1) expect(job_artifacts.count).to eq(1) - expect(File.exist?(@legacy_trace_path)).to be_falsy - expect(File.read(new_trace_path)).to eq('aiueo') + expect(File.exist?(legacy_trace_path(@build))).to be_falsy + expect(File.read(archived_trace_path(job_artifacts.first))).to eq('aiueo') end end context 'when trace file does not exsits at the right place' do - it 'correctly archive legacy traces' do - expect(job_artifacts.count).to eq(0) - expect(File.exist?(@legacy_trace_path)).to be_falsy - + it 'does not raise errors and create job artifact row' do described_class.new.perform(1, 1) expect(job_artifacts.count).to eq(0) end end - - def new_trace_path - job_artifact = job_artifacts.first - - disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) - creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') - - File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash, - creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log') - end end diff --git a/spec/migrations/archive_legacy_traces_spec.rb b/spec/migrations/archive_legacy_traces_spec.rb deleted file mode 100644 index fc61c4bec17..00000000000 --- a/spec/migrations/archive_legacy_traces_spec.rb +++ /dev/null @@ -1,45 +0,0 @@ -require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20180529152628_archive_legacy_traces') - -describe ArchiveLegacyTraces, :migration do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:builds) { table(:ci_builds) } - let(:job_artifacts) { table(:ci_job_artifacts) } - - before do - namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') - projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) - build = builds.create!(id: 1) - - @legacy_trace_path = File.join( - Settings.gitlab_ci.builds_path, - build.created_at.utc.strftime("%Y_%m"), - build.project_id.to_s, - "#{job.id}.log" - ) - - File.open(@legacy_trace_path, 'wb') { |stream| stream.write('aiueo') } - end - - it 'correctly archive legacy traces' do - expect(job_artifacts.count).to eq(0) - expect(File.exist?(@legacy_trace_path)).to be_truthy - - migrate! - - expect(job_artifacts.count).to eq(1) - expect(File.exist?(@legacy_trace_path)).to be_falsy - expect(File.exist?(new_trace_path)).to be_truthy - end - - def new_trace_path - job_artifact = job_artifacts.first - - disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) - creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') - - File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, - creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s) - end -end diff --git a/spec/migrations/schedule_to_archive_legacy_traces_spec.rb b/spec/migrations/schedule_to_archive_legacy_traces_spec.rb new file mode 100644 index 00000000000..d3eac3c45ea --- /dev/null +++ b/spec/migrations/schedule_to_archive_legacy_traces_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180529152628_schedule_to_archive_legacy_traces') + +describe ScheduleToArchiveLegacyTraces, :migration do + include TraceHelpers + + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:builds) { table(:ci_builds) } + let(:job_artifacts) { table(:ci_job_artifacts) } + + before do + namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) + @build_success = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build') + @build_failed = builds.create!(id: 2, project_id: 123, status: 'failed', type: 'Ci::Build') + @builds_canceled = builds.create!(id: 3, project_id: 123, status: 'canceled', type: 'Ci::Build') + @build_running = builds.create!(id: 4, project_id: 123, status: 'running', type: 'Ci::Build') + + create_legacy_trace(@build_success, 'This job is done') + create_legacy_trace(@build_failed, 'This job is done') + create_legacy_trace(@builds_canceled, 'This job is done') + create_legacy_trace(@build_running, 'This job is not done yet') + end + + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(File.exist?(legacy_trace_path(@build_success))).to be_truthy + expect(File.exist?(legacy_trace_path(@build_failed))).to be_truthy + expect(File.exist?(legacy_trace_path(@builds_canceled))).to be_truthy + expect(File.exist?(legacy_trace_path(@build_running))).to be_truthy + + migrate! + + expect(job_artifacts.count).to eq(3) + expect(File.exist?(legacy_trace_path(@build_success))).to be_falsy + expect(File.exist?(legacy_trace_path(@build_failed))).to be_falsy + expect(File.exist?(legacy_trace_path(@builds_canceled))).to be_falsy + expect(File.exist?(legacy_trace_path(@build_running))).to be_truthy + expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @build_success.id).first))).to be_truthy + expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @build_failed.id).first))).to be_truthy + expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @builds_canceled.id).first))).to be_truthy + expect(job_artifacts.where(job_id: @build_running.id)).not_to be_exist + end +end diff --git a/spec/support/trace/trace_helpers.rb b/spec/support/trace/trace_helpers.rb new file mode 100644 index 00000000000..f6d11b61038 --- /dev/null +++ b/spec/support/trace/trace_helpers.rb @@ -0,0 +1,23 @@ +module TraceHelpers + def create_legacy_trace(build, content) + File.open(legacy_trace_path(build), 'wb') { |stream| stream.write(content) } + end + + def legacy_trace_path(build) + legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, + build.created_at.utc.strftime("%Y_%m"), + build.project_id.to_s) + + FileUtils.mkdir_p(legacy_trace_dir) + + File.join(legacy_trace_dir, "#{build.id}.log") + end + + def archived_trace_path(job_artifact) + disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s) + creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d') + + File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash, + creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log') + end +end -- cgit v1.2.1 From 623accf9cb3babb81d789a013e6ad87ad0bf26e5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Jun 2018 15:09:49 +0900 Subject: Add type_build to ScheduleToArchiveLegacyTraces::Build --- db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb b/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb index ea782e1596b..965cd3a8714 100644 --- a/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb +++ b/db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb @@ -12,6 +12,8 @@ class ScheduleToArchiveLegacyTraces < ActiveRecord::Migration self.table_name = 'ci_builds' self.inheritance_column = :_type_disabled # Disable STI + scope :type_build, -> { where(type: 'Ci::Build') } + scope :finished, -> { where(status: [:success, :failed, :canceled]) } scope :without_archived_trace, -> do @@ -21,7 +23,7 @@ class ScheduleToArchiveLegacyTraces < ActiveRecord::Migration def up queue_background_migration_jobs_by_range_at_intervals( - ::ScheduleToArchiveLegacyTraces::Build.finished.without_archived_trace, + ::ScheduleToArchiveLegacyTraces::Build.type_build.finished.without_archived_trace, BACKGROUND_MIGRATION_CLASS, 5.minutes, batch_size: BATCH_SIZE) -- cgit v1.2.1 From 2184c753fd81925c74a3a30abe9be187aa8c4133 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Jun 2018 15:13:36 +0900 Subject: Revise comments in ArchiveLegacyTraces --- lib/gitlab/background_migration/archive_legacy_traces.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 0999ea95e56..5a4e5b2c471 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -6,10 +6,10 @@ module Gitlab module BackgroundMigration class ArchiveLegacyTraces def perform(start_id, stop_id) - # This background migrations directly refer ::Ci::Build model which is defined in application code. + # This background migration directly refers to ::Ci::Build model which is defined in application code. # In general, migration code should be isolated as much as possible in order to be idempotent. - # However, `archive!` logic is too complicated to be replicated. So we chose a way to refer ::Ci::Build directly - # and we don't change the `archive!` logic until 11.1 + # However, `archive!` method is too complicated to be replicated by coping its subsequent code. + # So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1 ::Ci::Build.finished.without_archived_trace .where(id: start_id..stop_id).find_each do |build| begin -- cgit v1.2.1 From 8f1f73d4e3ca82e3d449e478606f133d19ead7b1 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 4 Jun 2018 14:28:21 +0900 Subject: Fix typo in spec. Add a test for the case of when trace is stored in database --- .../archive_legacy_traces_spec.rb | 26 ++++++++++++++++++---- spec/support/trace/trace_helpers.rb | 4 ++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb index 0765f4149f9..877c061d11b 100644 --- a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb +++ b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb @@ -16,7 +16,7 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2 context 'when trace file exsits at the right place' do before do - create_legacy_trace(@build, 'aiueo') + create_legacy_trace(@build, 'trace in file') end it 'correctly archive legacy traces' do @@ -27,15 +27,33 @@ describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 2 expect(job_artifacts.count).to eq(1) expect(File.exist?(legacy_trace_path(@build))).to be_falsy - expect(File.read(archived_trace_path(job_artifacts.first))).to eq('aiueo') + expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in file') end end context 'when trace file does not exsits at the right place' do - it 'does not raise errors and create job artifact row' do - described_class.new.perform(1, 1) + it 'does not raise errors nor create job artifact' do + expect { described_class.new.perform(1, 1) }.not_to raise_error expect(job_artifacts.count).to eq(0) end end + + context 'when trace data exsits in database' do + before do + create_legacy_trace_in_db(@build, 'trace in db') + end + + it 'correctly archive legacy traces' do + expect(job_artifacts.count).to eq(0) + expect(@build.read_attribute(:trace)).not_to be_empty + + described_class.new.perform(1, 1) + + @build.reload + expect(job_artifacts.count).to eq(1) + expect(@build.read_attribute(:trace)).to be_nil + expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in db') + end + end end diff --git a/spec/support/trace/trace_helpers.rb b/spec/support/trace/trace_helpers.rb index f6d11b61038..c7802bbcb94 100644 --- a/spec/support/trace/trace_helpers.rb +++ b/spec/support/trace/trace_helpers.rb @@ -3,6 +3,10 @@ module TraceHelpers File.open(legacy_trace_path(build), 'wb') { |stream| stream.write(content) } end + def create_legacy_trace_in_db(build, content) + build.update_column(:trace, content) + end + def legacy_trace_path(build) legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path, build.created_at.utc.strftime("%Y_%m"), -- cgit v1.2.1 From 4bfd208b9f1738a67cac149ccac7ec2153c43448 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 4 Jun 2018 08:31:03 +0200 Subject: Bump gitlab-shell to 7.1.3 This includes the change that prints the @username of a user instead of the full name. https://gitlab.com/gitlab-org/gitlab-shell/merge_requests/204 --- GITLAB_SHELL_VERSION | 2 +- changelogs/unreleased/bvl-bump-gitlab-shell-7-1-3.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/bvl-bump-gitlab-shell-7-1-3.yml diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index a8a18875682..1996c504476 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -7.1.2 +7.1.3 diff --git a/changelogs/unreleased/bvl-bump-gitlab-shell-7-1-3.yml b/changelogs/unreleased/bvl-bump-gitlab-shell-7-1-3.yml new file mode 100644 index 00000000000..76bb25bc7d7 --- /dev/null +++ b/changelogs/unreleased/bvl-bump-gitlab-shell-7-1-3.yml @@ -0,0 +1,5 @@ +--- +title: Include username in output when testing SSH to GitLab +merge_request: 19358 +author: +type: other -- cgit v1.2.1 From 8008442829ea797b822d7c782334f6b2d319ca86 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Mon, 4 Jun 2018 08:30:31 +0000 Subject: Use gitlab-shell 7.1.4 --- GITLAB_SHELL_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index a8a18875682..b7f8ee41e69 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -7.1.2 +7.1.4 -- cgit v1.2.1 From 20b04cc07eab0b0d6108a1bb56f4575b92720b95 Mon Sep 17 00:00:00 2001 From: Mike Lewis Date: Mon, 4 Jun 2018 07:54:25 +0000 Subject: Update GitHub import instructions (github.md) --- doc/user/project/import/github.md | 172 ++++++++++++++------------------------ 1 file changed, 61 insertions(+), 111 deletions(-) diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 8c639bd5343..6757ba7061d 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -1,154 +1,104 @@ # Import your project from GitHub to GitLab -Import your projects from GitHub to GitLab with minimal effort. - ## Overview ->**Note:** -If you are an administrator you can enable the [GitHub integration][gh-import] -in your GitLab instance sitewide. This configuration is optional, users will -still be able to import their GitHub repositories with a -[personal access token][gh-token]. - ->**Note:** -Administrators of a GitLab instance (Community or Enterprise Edition) can also -use the [GitHub rake task][gh-rake] to import projects from GitHub without the -constrains of a Sidekiq worker. - -- At its current state, GitHub importer can import: - - the repository description (GitLab 7.7+) - - the Git repository data (GitLab 7.7+) - - the issues (GitLab 7.7+) - - the pull requests (GitLab 8.4+) - - the wiki pages (GitLab 8.4+) - - the milestones (GitLab 8.7+) - - the labels (GitLab 8.7+) - - the release note descriptions (GitLab 8.12+) - - the pull request review comments (GitLab 10.2+) - - the regular issue and pull request comments -- References to pull requests and issues are preserved (GitLab 8.7+) -- Repository public access is retained. If a repository is private in GitHub - it will be created as private in GitLab as well. +Using the importer, you can import your GitHub repositories to GitLab.com or to your self-hosted GitLab instance. -## How it works +>**Note:** While these instructions will always work for users on gitlab.com, if you are an administrator of a self-hosted GitLab instance, you will need to enable the [GitHub integration](https://docs.gitlab.com/ee/integration/github.html) in order for users to follow the preferred import method described on this page. If this is not enabled, users can alternatively import their GitHub repositories using a [personal access token](https://docs.gitlab.com/ee/user/project/import/github.html#authorize-access-to-your-repositories-using-a-personal-access-token) from GitHub, but this method will not be able to associate all user activity (such as issues and pull requests) with matching GitLab users. -When issues/pull requests are being imported, the GitHub importer tries to find -the GitHub author/assignee in GitLab's database using the GitHub ID. For this -to work, the GitHub author/assignee should have signed in beforehand in GitLab -and **associated their GitHub account**. If the user is not -found in GitLab's database, the project creator (most of the times the current -user that started the import process) is set as the author, but a reference on -the issue about the original GitHub author is kept. +>As an administrator of a self-hosted GitLab instance, you can also use the [GitHub rake task](https://docs.gitlab.com/ee/administration/raketasks/github_import.html) to import projects from GitHub without the constraints of a Sidekiq worker. -The importer will create any new namespaces (groups) if they don't exist or in -the case the namespace is taken, the repository will be imported under the user's -namespace that started the import process. +* The following aspects of a project are imported: + * repository description (GitLab.com & 7.7+) + * Git repository data (GitLab.com & 7.7+) + * issues (GitLab.com & 7.7+) + * pull requests (GitLab.com & 8.4+) + * wiki pages (GitLab.com & 8.4+) + * milestones (GitLab.com & 8.7+) + * labels (GitLab.com & 8.7+) + * release note descriptions (GitLab.com & 8.12+) + * pull request review comments (GitLab.com & 10.2+) + * regular issue and pull request comments +* References to pull requests and issues are preserved (GitLab.com & 8.7+) +* Each imported repository defaults to ‘private’ but can be made public, as needed. -The importer will also import branches on forks of projects related to open pull -requests. These branches will be imported with a naming scheme similar to -GH-SHA-Username/Pull-Request-number/fork-name/branch. This may lead to a discrepancy -in branches compared to the GitHub Repository. +## How it works -For a more technical description and an overview of the architecture you can -refer to [Working with the GitHub importer][gh-import-dev-docs]. +When issues and pull requests are being imported, the importer attempts to find their GitHub authors and assignees in the database of the GitLab instance. (Note that pull requests are called "merge requests" in GitLab.) -## Importing your GitHub repositories +For this association to succeed, prior to the import, each GitHub author and assignee in the repository must have either previously logged in to a GitLab account using the GitHub icon **or** have a GitHub account with a [public email address](https://help.github.com/articles/setting-your-commit-email-address-on-github/) that matches their GitLab account’s email address. -The importer page is visible when you create a new project. +If a user referenced in the project is not found in GitLab's database, the project creator (typically the user that initiated the import process) is set as the author/assignee, but a note on the issue mentioning the original GitHub author is added. -![New project page on GitLab](img/import_projects_from_new_project_page.png) +The importer creates any new namespaces (groups) if they do not exist, or, if the namespace is taken, the repository is imported under the namespace of the user who initiated the import process. The namespace/repository name can also be edited, with the proper permissions. -Click on the **GitHub** link and the import authorization process will start. -There are two ways to authorize access to your GitHub repositories: +The importer will also import branches on forks of projects related to open pull requests. These branches will be imported with a naming scheme similar to GH-`SHA-Username/Pull-Request-number/fork-name/branch`. This may lead to a discrepancy in branches compared to those of the GitHub repository. -1. [Using the GitHub integration][gh-integration] (if it's enabled by your - GitLab administrator). This is the preferred way as it's possible to - preserve the GitHub authors/assignees. Read more in the [How it works](#how-it-works) - section. -1. [Using a personal access token][gh-token] provided by GitHub. +For additional technical details, you can refer to the [GitHub Importer][gh-import-dev-docs] developer documentation. -![Select authentication method](img/import_projects_from_github_select_auth_method.png) +## Import your GitHub repository into GitLab -### Authorize access to your repositories using the GitHub integration +### Use the GitHub integration -If the [GitHub integration][gh-import] is enabled by your GitLab administrator, -you can use it instead of the personal access token. +Before you begin, ensure that any GitHub users who you want to map to GitLab users have either: +1. A GitLab account that has logged in using the GitHub icon. +\- or - +2. A GitLab account with an email address that matches the [public email address](https://help.github.com/articles/setting-your-commit-email-address-on-github/) of the GitHub user. -1. First you may want to connect your GitHub account to GitLab in order for - the username mapping to be correct. -1. Once you connect GitHub, click the **List your GitHub repositories** button - and you will be redirected to GitHub for permission to access your projects. -1. After accepting, you'll be automatically redirected to the importer. +User-matching attempts occur in that order, and if a user is not identified either way, the activity is associated with the user account that is performing the import. -You can now go on and [select which repositories to import](#select-which-repositories-to-import). +Note: If you are using a self-hosted GitLab instance, this process requires that you have configured the [GitHub integration][gh-import]. -### Authorize access to your repositories using a personal access token +1. From the top navigation bar, click **+** and select **New Project**. +2. Select the **Import project** tab and then select **GitHub**. +3. Select the first button to **List your GitHub repositories**. You are redirected to a page on github.com to authorize the GitLab application. +4. Click **Authorize gitlabhq**. You are redirected back to GitLab’s Import page and all of your GitHub repositories are listed. +5. Continue on to [select which repositories to import](#select-which-repositories-to-import). ->**Note:** -For a proper author/assignee mapping for issues and pull requests, the -[GitHub integration][gh-integration] should be used instead of the -[personal access token][gh-token]. If the GitHub integration is enabled by your -GitLab administrator, it should be the preferred method to import your repositories. -Read more in the [How it works](#how-it-works) section. +### Use a GitHub token -If you are not using the GitHub integration, you can still perform a one-off -authorization with GitHub to grant GitLab access your repositories: +>**Note:** For a proper author/assignee mapping for issues and pull requests, the GitHub integration method (above) should be used instead of the personal access token. If you are using gitlab.com or a self-hosted GitLab instance with the GitHub integration enabled, that should be the preferred method to import your repositories. Read more in the [How it works](#how-it-works) section. -1. Go to . -1. Enter a token description. -1. Check the `repo` scope. -1. Click **Generate token**. -1. Copy the token hash. -1. Go back to GitLab and provide the token to the GitHub importer. -1. Hit the **List Your GitHub Repositories** button and wait while GitLab reads - your repositories' information. Once done, you'll be taken to the importer - page to select the repositories to import. +If you are not using the GitHub integration, you can still perform an authorization with GitHub to grant GitLab access your repositories: -### Select which repositories to import +1. Go to [https://github.com/settings/tokens/new](https://github.com/settings/tokens/new). +2. Enter a token description. +3. Select the repo scope. +4. Click **Generate token**. +5. Copy the token hash. +6. Go back to GitLab and provide the token to the GitHub importer. +7. Hit the **List Your GitHub Repositories** button and wait while GitLab reads your repositories' information. Once done, you'll be taken to the importer page to select the repositories to import. -After you've authorized access to your GitHub repositories, you will be -redirected to the GitHub importer page. +### Select which repositories to import -From there, you can see the import statuses of your GitHub repositories. +After you have authorized access to your GitHub repositories, you are redirected to the GitHub importer page and your GitHub repositories are listed. -- Those that are being imported will show a _started_ status, -- those already successfully imported will be green with a _done_ status, -- whereas those that are not yet imported will have an **Import** button on the - right side of the table. +1. By default, the proposed repository namespaces match the names as they exist in GitHub, but based on your permissions, you can choose to edit these names before you proceed to import any of them. +2. Select the **Import** button next to any number of repositories, or select **Import all repositories**. +3. The **Status** column shows the import status of each repository. You can choose to leave the page open and it will update in realtime or you can return to it later. +4. Once a repository has been imported, click its GitLab path to open its GitLab URL. -If you want, you can import all your GitHub projects in one go by hitting -**Import all projects** in the upper left corner. +## Mirroring and pipeline status sharing -![GitHub importer page](img/import_projects_from_github_importer.png) +Depending your GitLab tier, [project mirroring](../../../workflow/repository_mirroring.md) can be set up to keep your imported project in sync with its GitHub copy. ---- +Additionally you can configure GitLab to send pipeline status updates back GitHub with the [GitHub Project Integration](../integrations/github.md). -You can also choose a different name for the project and a different namespace, -if you have the privileges to do so. +If you import your project using [CI/CD for external repo](../../../ci/ci_cd_for_external_repos/) then both of the above are automatically configured. -## Making the import process go faster +## Improving the speed of imports on self-hosted instances -For large projects it may take a while to import all data. To reduce the time -necessary you can increase the number of Sidekiq workers that process the -following queues: +For large projects it may take a while to import all data. To reduce the time necessary you can increase the number of Sidekiq workers that process the following queues: * `github_importer` * `github_importer_advance_stage` -For an optimal experience we recommend having at least 4 Sidekiq processes (each -running a number of threads equal to the number of CPU cores) that _only_ -process these queues. We also recommend that these processes run on separate -servers. For 4 servers with 8 cores this means you can import up to 32 objects -(e.g. issues) in parallel. +For an optimal experience we recommend having at least 4 Sidekiq processes (each running a number of threads equal to the number of CPU cores) that *only* process these queues. We also recommend that these processes run on separate servers. For 4 servers with 8 cores this means you can import up to 32 objects (e.g. issues) in parallel. -Reducing the time spent in cloning a repository can be done by increasing -network throughput, CPU capacity, and disk performance (e.g. by using high -performance SSDs) of the disks that store the Git repositories (for your GitLab -instance). Increasing the number of Sidekiq workers will _not_ reduce the time -spent cloning repositories. +Reducing the time spent in cloning a repository can be done by increasing network throughput, CPU capacity, and disk performance (e.g. by using high performance SSDs) of the disks that store the Git repositories (for your GitLab instance). Increasing the number of Sidekiq workers will *not* reduce the time spent cloning repositories. [gh-import]: ../../../integration/github.md "GitHub integration" [gh-rake]: ../../../administration/raketasks/github_import.md "GitHub rake task" [gh-integration]: #authorize-access-to-your-repositories-using-the-github-integration [gh-token]: #authorize-access-to-your-repositories-using-a-personal-access-token -[gh-import-dev-docs]: ../../../development/github_importer.md "Working with the GitHub importer" +[gh-import-dev-docs]: ../../../development/github_importer.md "Working with the GitHub importer" \ No newline at end of file -- cgit v1.2.1 From 8197c756b13be9189eebb649fdb61e7aebb7978d Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Mon, 4 Jun 2018 11:12:21 +0200 Subject: Copyedit GitHub importer docs --- doc/integration/github.md | 2 +- doc/user/project/import/github.md | 165 +++++++++++++++++++++++--------------- 2 files changed, 103 insertions(+), 64 deletions(-) diff --git a/doc/integration/github.md b/doc/integration/github.md index 23bb8ef9303..680712f9e01 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -110,7 +110,7 @@ On the sign in page there should now be a GitHub icon below the regular sign in Click the icon to begin the authentication process. GitHub will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. -### GitHub Enterprise with Self-Signed Certificate +## GitHub Enterprise with self-signed Certificate If you are attempting to import projects from GitHub Enterprise with a self-signed certificate and the imports are failing, you will need to disable SSL verification. diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 6757ba7061d..cad85881c4d 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -1,104 +1,143 @@ # Import your project from GitHub to GitLab -## Overview - -Using the importer, you can import your GitHub repositories to GitLab.com or to your self-hosted GitLab instance. - ->**Note:** While these instructions will always work for users on gitlab.com, if you are an administrator of a self-hosted GitLab instance, you will need to enable the [GitHub integration](https://docs.gitlab.com/ee/integration/github.html) in order for users to follow the preferred import method described on this page. If this is not enabled, users can alternatively import their GitHub repositories using a [personal access token](https://docs.gitlab.com/ee/user/project/import/github.html#authorize-access-to-your-repositories-using-a-personal-access-token) from GitHub, but this method will not be able to associate all user activity (such as issues and pull requests) with matching GitLab users. - ->As an administrator of a self-hosted GitLab instance, you can also use the [GitHub rake task](https://docs.gitlab.com/ee/administration/raketasks/github_import.html) to import projects from GitHub without the constraints of a Sidekiq worker. - -* The following aspects of a project are imported: - * repository description (GitLab.com & 7.7+) - * Git repository data (GitLab.com & 7.7+) - * issues (GitLab.com & 7.7+) - * pull requests (GitLab.com & 8.4+) - * wiki pages (GitLab.com & 8.4+) - * milestones (GitLab.com & 8.7+) - * labels (GitLab.com & 8.7+) - * release note descriptions (GitLab.com & 8.12+) - * pull request review comments (GitLab.com & 10.2+) - * regular issue and pull request comments -* References to pull requests and issues are preserved (GitLab.com & 8.7+) -* Each imported repository defaults to ‘private’ but can be made public, as needed. - -## How it works - -When issues and pull requests are being imported, the importer attempts to find their GitHub authors and assignees in the database of the GitLab instance. (Note that pull requests are called "merge requests" in GitLab.) - -For this association to succeed, prior to the import, each GitHub author and assignee in the repository must have either previously logged in to a GitLab account using the GitHub icon **or** have a GitHub account with a [public email address](https://help.github.com/articles/setting-your-commit-email-address-on-github/) that matches their GitLab account’s email address. +Using the importer, you can import your GitHub repositories to GitLab.com or to +your self-hosted GitLab instance. -If a user referenced in the project is not found in GitLab's database, the project creator (typically the user that initiated the import process) is set as the author/assignee, but a note on the issue mentioning the original GitHub author is added. - -The importer creates any new namespaces (groups) if they do not exist, or, if the namespace is taken, the repository is imported under the namespace of the user who initiated the import process. The namespace/repository name can also be edited, with the proper permissions. - -The importer will also import branches on forks of projects related to open pull requests. These branches will be imported with a naming scheme similar to GH-`SHA-Username/Pull-Request-number/fork-name/branch`. This may lead to a discrepancy in branches compared to those of the GitHub repository. - -For additional technical details, you can refer to the [GitHub Importer][gh-import-dev-docs] developer documentation. - -## Import your GitHub repository into GitLab +## Overview -### Use the GitHub integration +NOTE: **Note:** +While these instructions will always work for users on GitLab.com, if you are an +administrator of a self-hosted GitLab instance, you will need to enable the +[GitHub integration][gh-import] in order for users to follow the preferred +import method described on this page. If this is not enabled, users can alternatively import their +GitHub repositories using a [personal access token](#using-a-github-token) from GitHub, +but this method will not be able to associate all user activity (such as issues and pull requests) +with matching GitLab users. As an administrator of a self-hosted GitLab instance, you can also use +the [GitHub rake task](../../../administration/raketasks/github_import.md) to import projects from +GitHub without the constraints of a Sidekiq worker. + +The following aspects of a project are imported: + * Repository description (GitLab.com & 7.7+) + * Git repository data (GitLab.com & 7.7+) + * Issues (GitLab.com & 7.7+) + * Pull requests (GitLab.com & 8.4+) + * Wiki pages (GitLab.com & 8.4+) + * Milestones (GitLab.com & 8.7+) + * Labels (GitLab.com & 8.7+) + * Release note descriptions (GitLab.com & 8.12+) + * Pull request review comments (GitLab.com & 10.2+) + * Regular issue and pull request comments + +References to pull requests and issues are preserved (GitLab.com & 8.7+), and +each imported repository defaults to `private` but [can be made public](../settings/index.md#sharing-and-permissions), as needed. + +## How it works + +When issues and pull requests are being imported, the importer attempts to find their GitHub authors and +assignees in the database of the GitLab instance (note that pull requests are called "merge requests" in GitLab). + +For this association to succeed, prior to the import, each GitHub author and assignee in the repository must +have either previously logged in to a GitLab account using the GitHub icon **or** have a GitHub account with +a [public email address](https://help.github.com/articles/setting-your-commit-email-address-on-github/) that +matches their GitLab account's email address. + +If a user referenced in the project is not found in GitLab's database, the project creator (typically the user +that initiated the import process) is set as the author/assignee, but a note on the issue mentioning the original +GitHub author is added. + +The importer creates any new namespaces (groups) if they do not exist, or, if the namespace is taken, the +repository is imported under the namespace of the user who initiated the import process. The namespace/repository +name can also be edited, with the proper permissions. + +The importer will also import branches on forks of projects related to open pull requests. These branches will be +imported with a naming scheme similar to `GH-SHA-username/pull-request-number/fork-name/branch`. This may lead to +a discrepancy in branches compared to those of the GitHub repository. + +For additional technical details, you can refer to the +[GitHub Importer](../../../development/github_importer.md "Working with the GitHub importer") +developer documentation. + +## Import your GitHub repository into GitLab + +### Using the GitHub integration Before you begin, ensure that any GitHub users who you want to map to GitLab users have either: -1. A GitLab account that has logged in using the GitHub icon. + +1. A GitLab account that has logged in using the GitHub icon \- or - -2. A GitLab account with an email address that matches the [public email address](https://help.github.com/articles/setting-your-commit-email-address-on-github/) of the GitHub user. +2. A GitLab account with an email address that matches the [public email address](https://help.github.com/articles/setting-your-commit-email-address-on-github/) of the GitHub user -User-matching attempts occur in that order, and if a user is not identified either way, the activity is associated with the user account that is performing the import. +User-matching attempts occur in that order, and if a user is not identified either way, the activity is associated with +the user account that is performing the import. -Note: If you are using a self-hosted GitLab instance, this process requires that you have configured the [GitHub integration][gh-import]. +NOTE: **Note:** +If you are using a self-hosted GitLab instance, this process requires that you have configured the +[GitHub integration][gh-import]. -1. From the top navigation bar, click **+** and select **New Project**. +1. From the top navigation bar, click **+** and select **New project**. 2. Select the **Import project** tab and then select **GitHub**. 3. Select the first button to **List your GitHub repositories**. You are redirected to a page on github.com to authorize the GitLab application. -4. Click **Authorize gitlabhq**. You are redirected back to GitLab’s Import page and all of your GitHub repositories are listed. -5. Continue on to [select which repositories to import](#select-which-repositories-to-import). +4. Click **Authorize gitlabhq**. You are redirected back to GitLab's Import page and all of your GitHub repositories are listed. +5. Continue on to [selecting which repositories to import](#selecting-which-repositories-to-import). -### Use a GitHub token +### Using a GitHub token ->**Note:** For a proper author/assignee mapping for issues and pull requests, the GitHub integration method (above) should be used instead of the personal access token. If you are using gitlab.com or a self-hosted GitLab instance with the GitHub integration enabled, that should be the preferred method to import your repositories. Read more in the [How it works](#how-it-works) section. +NOTE: **Note:** +For a proper author/assignee mapping for issues and pull requests, the [GitHub integration method (above)](#using-the-github-integration) +should be used instead of the personal access token. If you are using GitLab.com or a self-hosted GitLab instance with the GitHub +integration enabled, that should be the preferred method to import your repositories. Read more in the [How it works](#how-it-works) section. If you are not using the GitHub integration, you can still perform an authorization with GitHub to grant GitLab access your repositories: -1. Go to [https://github.com/settings/tokens/new](https://github.com/settings/tokens/new). +1. Go to https://github.com/settings/tokens/new 2. Enter a token description. 3. Select the repo scope. 4. Click **Generate token**. 5. Copy the token hash. 6. Go back to GitLab and provide the token to the GitHub importer. -7. Hit the **List Your GitHub Repositories** button and wait while GitLab reads your repositories' information. Once done, you'll be taken to the importer page to select the repositories to import. +7. Hit the **List Your GitHub Repositories** button and wait while GitLab reads your repositories' information. + Once done, you'll be taken to the importer page to select the repositories to import. -### Select which repositories to import +### Selecting which repositories to import -After you have authorized access to your GitHub repositories, you are redirected to the GitHub importer page and your GitHub repositories are listed. +After you have authorized access to your GitHub repositories, you are redirected to the GitHub importer page and +your GitHub repositories are listed. -1. By default, the proposed repository namespaces match the names as they exist in GitHub, but based on your permissions, you can choose to edit these names before you proceed to import any of them. +1. By default, the proposed repository namespaces match the names as they exist in GitHub, but based on your permissions, + you can choose to edit these names before you proceed to import any of them. 2. Select the **Import** button next to any number of repositories, or select **Import all repositories**. -3. The **Status** column shows the import status of each repository. You can choose to leave the page open and it will update in realtime or you can return to it later. +3. The **Status** column shows the import status of each repository. You can choose to leave the page open and it will + update in realtime or you can return to it later. 4. Once a repository has been imported, click its GitLab path to open its GitLab URL. ## Mirroring and pipeline status sharing -Depending your GitLab tier, [project mirroring](../../../workflow/repository_mirroring.md) can be set up to keep your imported project in sync with its GitHub copy. +Depending your GitLab tier, [project mirroring](../../../workflow/repository_mirroring.md) can be set up to keep +your imported project in sync with its GitHub copy. + +Additionally, you can configure GitLab to send pipeline status updates back GitHub with the +[GitHub Project Integration](https://docs.gitlab.com/ee/user/project/integrations/github.html). **[PREMIUM]** -Additionally you can configure GitLab to send pipeline status updates back GitHub with the [GitHub Project Integration](../integrations/github.md). +If you import your project using [CI/CD for external repo](https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/), then both +of the above are automatically configured. **[PREMIUM]** -If you import your project using [CI/CD for external repo](../../../ci/ci_cd_for_external_repos/) then both of the above are automatically configured. +## Improving the speed of imports on self-hosted instances -## Improving the speed of imports on self-hosted instances +NOTE: **Note:** +Admin access to the GitLab server is required. -For large projects it may take a while to import all data. To reduce the time necessary you can increase the number of Sidekiq workers that process the following queues: +For large projects it may take a while to import all data. To reduce the time necessary, you can increase the number of +Sidekiq workers that process the following queues: * `github_importer` * `github_importer_advance_stage` -For an optimal experience we recommend having at least 4 Sidekiq processes (each running a number of threads equal to the number of CPU cores) that *only* process these queues. We also recommend that these processes run on separate servers. For 4 servers with 8 cores this means you can import up to 32 objects (e.g. issues) in parallel. +For an optimal experience, it's recommended having at least 4 Sidekiq processes (each running a number of threads equal +to the number of CPU cores) that *only* process these queues. It's also recommended that these processes run on separate +servers. For 4 servers with 8 cores this means you can import up to 32 objects (e.g., issues) in parallel. -Reducing the time spent in cloning a repository can be done by increasing network throughput, CPU capacity, and disk performance (e.g. by using high performance SSDs) of the disks that store the Git repositories (for your GitLab instance). Increasing the number of Sidekiq workers will *not* reduce the time spent cloning repositories. +Reducing the time spent in cloning a repository can be done by increasing network throughput, CPU capacity, and disk +performance (e.g., by using high performance SSDs) of the disks that store the Git repositories (for your GitLab instance). +Increasing the number of Sidekiq workers will *not* reduce the time spent cloning repositories. [gh-import]: ../../../integration/github.md "GitHub integration" -[gh-rake]: ../../../administration/raketasks/github_import.md "GitHub rake task" -[gh-integration]: #authorize-access-to-your-repositories-using-the-github-integration -[gh-token]: #authorize-access-to-your-repositories-using-a-personal-access-token -[gh-import-dev-docs]: ../../../development/github_importer.md "Working with the GitHub importer" \ No newline at end of file -- cgit v1.2.1 From 2d7aef1397c08f6fade604cdecfe542b7088ac26 Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Mon, 4 Jun 2018 11:15:48 +0200 Subject: Port repository mirroring from EE --- doc/workflow/repository_mirroring.md | 250 ++++++++++++++++++++- .../repository_mirroring_detect_host_keys.png | Bin 0 -> 61463 bytes .../repository_mirroring_diverged_branch.png | Bin 0 -> 22668 bytes .../repository_mirroring_hard_failed_main.png | Bin 0 -> 47943 bytes .../repository_mirroring_hard_failed_settings.png | Bin 0 -> 53279 bytes .../repository_mirroring_new_project.png | Bin 0 -> 20364 bytes ...epository_mirroring_pull_advanced_host_keys.png | Bin 0 -> 115796 bytes .../repository_mirroring_pull_settings.png | Bin 0 -> 100470 bytes .../repository_mirroring_pull_settings_for_ssh.png | Bin 0 -> 69467 bytes ...repository_mirroring_ssh_host_keys_verified.png | Bin 0 -> 23724 bytes ...ory_mirroring_ssh_public_key_authentication.png | Bin 0 -> 82456 bytes 11 files changed, 245 insertions(+), 5 deletions(-) create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_detect_host_keys.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_diverged_branch.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_hard_failed_main.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_hard_failed_settings.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_new_project.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_pull_advanced_host_keys.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_pull_settings.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_pull_settings_for_ssh.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_ssh_host_keys_verified.png create mode 100644 doc/workflow/repository_mirroring/repository_mirroring_ssh_public_key_authentication.png diff --git a/doc/workflow/repository_mirroring.md b/doc/workflow/repository_mirroring.md index dbe63144e38..aaddbe4fbf5 100644 --- a/doc/workflow/repository_mirroring.md +++ b/doc/workflow/repository_mirroring.md @@ -1,6 +1,6 @@ # Repository mirroring -Repository Mirroring is a way to mirror repositories from external sources. +Repository mirroring is a way to mirror repositories from external sources. It can be used to mirror all branches, tags, and commits that you have in your repository. @@ -34,13 +34,200 @@ A few things/limitations to consider: - The Git LFS objects will not be synced. You'll need to push/pull them manually. -## Use-case +## Use cases +- You migrated to GitLab but still need to keep your project in another source. + In that case, you can simply set it up to mirror to GitLab (pull) and all the + essential history of commits, tags and branches will be available in your + GitLab instance. - You have old projects in another source that you don't use actively anymore, but don't want to remove for archiving purposes. In that case, you can create a push mirror so that your active GitLab repository can push its changes to the old location. +## Pulling from a remote repository **[STARTER]** + +>[Introduced][ee-51] in GitLab Enterprise Edition 8.2. + +You can set up a repository to automatically have its branches, tags, and commits +updated from an upstream repository. This is useful when a repository you're +interested in is located on a different server, and you want to be able to +browse its content and its activity using the familiar GitLab interface. + +When creating a new project, you can enable repository mirroring when you choose +to import the repository from "Any repo by URL". Enter the full URL of the Git +repository to pull from and click on the **Mirror repository** checkbox. + +![New project](repository_mirroring/repository_mirroring_new_project.png) + +For an existing project, you can set up mirror pulling by visiting your project's +**Settings ➔ Repository** and searching for the "Pull from a remote repository" +section. Check the "Mirror repository" box and hit **Save changes** at the bottom. +You have a few options to choose from one being the user who will be the author +of all events in the activity feed that are the result of an update. This user +needs to have at least [master access][perms] to the project. Another option is +whether you want to trigger builds for mirror updates. + +![Pull settings](repository_mirroring/repository_mirroring_pull_settings.png) + +Since the repository on GitLab functions as a mirror of the upstream repository, +you are advised not to push commits directly to the repository on GitLab. +Instead, any commits should be pushed to the upstream repository, and will end +up in the GitLab repository automatically within a certain period of time +or when a [forced update](#forcing-an-update) is initiated. + +If you do manually update a branch in the GitLab repository, the branch will +become diverged from upstream, and GitLab will no longer automatically update +this branch to prevent any changes from being lost. + +![Diverged branch](repository_mirroring/repository_mirroring_diverged_branch.png) + +### Trigger update using API **[STARTER]** + +>[Introduced][ee-3453] in GitLab Enterprise Edition 10.3. + +Pull mirroring uses polling to detect new branches and commits added upstream, +often many minutes afterwards. If you notify GitLab by [API][pull-api], updates +will be pulled immediately. + +Read the [Pull Mirror Trigger API docs][pull-api]. + +### Pull only protected branches **[STARTER]** + +>[Introduced][ee-3326] in GitLab Enterprise Edition 10.3. + +You can choose to only pull the protected branches from your remote repository to GitLab. + +To use this option go to your project's repository settings page under pull mirror. + +### Overwrite diverged branches **[STARTER]** + +>[Introduced][ee-4559] in GitLab Enterprise Edition 10.6. + +You can choose to always update your local branch with the remote version even +if your local version has diverged from the remote. + +To use this option go to your project's repository settings page under pull mirror. + +### Hard failure **[STARTER]** + +>[Introduced][ee-3117] in GitLab Enterprise Edition 10.2. + +Once a mirror gets retried 14 times in a row, it will get marked as hard failed, +this will become visible in either the project main dashboard or in the +pull mirror settings page. + +![Hard failed mirror main notice](repository_mirroring/repository_mirroring_hard_failed_main.png) + +![Hard failed mirror settings notice](repository_mirroring/repository_mirroring_hard_failed_settings.png) + +When a project is hard failed, it will no longer get picked up for mirroring. +A user can resume the project mirroring again by either [forcing an update](#forcing-an-update) +or by changing the import URL in repository settings. + +### SSH authentication **[STARTER]** + +> [Introduced][ee-2551] in GitLab Starter 9.5 + +If you're mirroring over SSH (i.e., an `ssh://` URL), you can authenticate using +password-based authentication, just as over HTTPS, but you can also use public +key authentication. This is often more secure than password authentication, +especially when the source repository supports [Deploy Keys][deploy-key]. + +To get started, navigate to **Settings ➔ Repository ➔ Pull from a remote repository**, +enable mirroring (if not already enabled) and enter an `ssh://` URL. + +> **NOTE**: SCP-style URLs, e.g., `git@example.com:group/project.git`, are not +supported at this time. + +Entering the URL adds two features to the page - `Fingerprints` and +`SSH public key authentication`: + +![Pull settings for SSH](repository_mirroring/repository_mirroring_pull_settings_for_ssh.png) + +SSH authentication is mutual. You have to prove to the server that you're +allowed to access the repository, but the server also has to prove to *you* that +it's who it claims to be. You provide your credentials as a password or public +key. The server that the source repository resides on provides its credentials +as a "host key", the fingerprint of which needs to be verified manually. + +Press the `Detect host keys` button. GitLab will fetch the host keys from the +server, and display the fingerprints to you: + +![Detect SSH host keys](repository_mirroring/repository_mirroring_detect_host_keys.png) + +You now need to verify that the fingerprints are those you expect. GitLab.com +and other code hosting sites publish their fingerprints in the open for you +to check: + +* [AWS CodeCommit](http://docs.aws.amazon.com/codecommit/latest/userguide/regions.html#regions-fingerprints) +* [Bitbucket](https://confluence.atlassian.com/bitbucket/use-the-ssh-protocol-with-bitbucket-cloud-221449711.html#UsetheSSHprotocolwithBitbucketCloud-KnownhostorBitbucket%27spublickeyfingerprints) +* [GitHub](https://help.github.com/articles/github-s-ssh-key-fingerprints/) +* [GitLab.com](https://about.gitlab.com/gitlab-com/settings/#ssh-host-keys-fingerprints) +* [Launchpad](https://help.launchpad.net/SSHFingerprints) +* [Savannah](http://savannah.gnu.org/maintenance/SshAccess/) +* [SourceForge](https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/) + +Other providers will vary. If you're running on-premises GitLab, or otherwise +have access to the source server, you can securely gather the key fingerprints: + +``` +$ cat /etc/ssh/ssh_host*pub | ssh-keygen -E md5 -l -f - +256 MD5:f4:28:9f:23:99:15:21:1b:bf:ed:1f:8e:a0:76:b2:9d root@example.com (ECDSA) +256 MD5:e6:eb:45:8a:3c:59:35:5f:e9:5b:80:12:be:7e:22:73 root@example.com (ED25519) +2048 MD5:3f:72:be:3d:62:03:5c:62:83:e8:6e:14:34:3a:85:1d root@example.com (RSA) +``` + +(You may need to exclude `-E md5` for some older versions of SSH). + +If you're an SSH expert and already have a `known_hosts` file you'd like to use +unaltered, then you can skip these steps. Just press the "Show advanced" button +and paste in the file contents: + +![Advanced SSH host key management](repository_mirroring/repository_mirroring_pull_advanced_host_keys.png) + +Once you've **carefully verified** that all the fingerprints match your trusted +source, you can press `Save changes`. This will record the host keys, along with +the person who verified them (you!) and the date: + +![SSH host keys submitted](repository_mirroring/repository_mirroring_ssh_host_keys_verified.png) + +When pulling changes from the source repository, GitLab will now check that at +least one of the stored host keys matches before connecting. This can prevent +malicious code from being injected into your mirror, or your password being +stolen! + +To use SSH public key authentication, you'll also need to choose that option +from the authentication methods dropdown. GitLab will generate a 4096-bit RSA +key and display the public component of that key to you: + +![SSH public key authentication](repository_mirroring/repository_mirroring_ssh_public_key_authentication.png) + +You then need to add the public SSH key to the source repository configuration. +If the source is hosted on GitLab, you should add it as a [Deploy Key][deploy-key]. +Other sources may require you to add the key to your user's `authorized_keys` +file - just paste the entire `ssh-rsa AAA.... user@host` block into the file on +its own line and save it. + +Once the public key is set up on the source repository, press `Save changes` and your +mirror will begin working. + +If you need to change the key at any time, you can press the `Regenerate key` +button to do so. You'll have to update the source repository with the new key +to keep the mirror running. + +### How it works + +Once you activate the pull mirroring feature, the mirror will be inserted into +a queue. A scheduler will start every minute and schedule a fixed amount of +mirrors for update, based on the configured maximum capacity. + +If the mirror successfully updates it will be enqueued once again with a small +backoff period. + +If the mirror fails (eg: branch diverged from upstream), the project's backoff +period will be penalized each time it fails up to a maximum amount of time. + ## Pushing to a remote repository **[STARTER]** >[Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/249) in @@ -83,7 +270,7 @@ To use this option go to your project's repository settings page under push mirr To set up a mirror from GitLab to GitHub, you need to follow these steps: 1. Create a [GitHub personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) with the "public_repo" box checked: - + ![edit personal access token GitHub](repository_mirroring/repository_mirroring_github_edit_personal_access_token.png) 1. Fill in the "Git repository URL" with the personal access token replacing the password `https://GitHubUsername:GitHubPersonalAccessToken@github.com/group/project.git`: @@ -94,7 +281,7 @@ To set up a mirror from GitLab to GitHub, you need to follow these steps: 1. And either wait or trigger the "Update Now" button: ![update now](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png) - + ## Forcing an update While mirrors are scheduled to update automatically, you can always force an update @@ -105,7 +292,60 @@ by using the **Update now** button which is exposed in various places: - in the tags page - in the **Mirror repository** settings page +## Bidirectional mirroring + +CAUTION: **Warning:** +There is no bidirectional support without conflicts. If you +configure a repository to pull and push to a second remote, there is no +guarantee that it will update correctly on both remotes. If you configure +a repository for bidirectional mirroring, you should consider when conflicts +occur who and how they will be resolved. + +Rewriting any mirrored commit on either remote will cause conflicts and +mirroring to fail. This can be prevented by [only pulling protected branches]( +#pull-only-protected-branches) and [only pushing protected branches]( +#push-only-protected-branches). You should protect the branches you wish to +mirror on both remotes to prevent conflicts caused by rewriting history. + +Bidirectional mirroring also creates a race condition where commits to the same +branch in close proximity will cause conflicts. The race condition can be +mitigated by reducing the mirroring delay by using a Push event webhook to +trigger an immediate pull to GitLab. Push mirroring from GitLab is rate limited +to once per minute when only push mirroring protected branches. + +It may be possible to implement a locking mechanism using the server-side +`pre-receive` hook to prevent the race condition. Read about [configuring +custom Git hooks][hooks] on the GitLab server. + +### Mirroring with Perforce via GitFusion + +CAUTION: **Warning:** +Bidirectional mirroring should not be used as a permanent +configuration. There is no bidirectional mirroring without conflicts. +Refer to [Migrating from Perforce Helix][perforce] for alternative migration +approaches. + +GitFusion provides a Git interface to Perforce which can be used by GitLab to +bidirectionally mirror projects with GitLab. This may be useful in some +situations when migrating from Perforce to GitLab where overlapping Perforce +workspaces cannot be migrated simultaneously to GitLab. + +If using mirroring with Perforce you should only mirror protected branches. +Perforce will reject any pushes that rewrite history. It is recommended that +only the fewest number of branches are mirrored due to the performance +limitations of GitFusion. + +[ee-51]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/51 +[ee-2551]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2551 +[ee-3117]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3117 +[ee-3326]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3326 [ee-3350]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3350 +[ee-3453]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3453 +[ee-4559]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/4559 [ce-18715]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18715 [perms]: ../user/permissions.md - +[hooks]: ../administration/custom_hooks.md +[deploy-key]: ../ssh/README.md#deploy-keys +[webhook]: ../user/project/integrations/webhooks.md#push-events +[pull-api]: ../api/projects.md#start-the-pull-mirroring-process-for-a-project +[perforce]: ../user/project/import/perforce.md diff --git a/doc/workflow/repository_mirroring/repository_mirroring_detect_host_keys.png b/doc/workflow/repository_mirroring/repository_mirroring_detect_host_keys.png new file mode 100644 index 00000000000..333648942f8 Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_detect_host_keys.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_diverged_branch.png b/doc/workflow/repository_mirroring/repository_mirroring_diverged_branch.png new file mode 100644 index 00000000000..45c9bce0889 Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_diverged_branch.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_hard_failed_main.png b/doc/workflow/repository_mirroring/repository_mirroring_hard_failed_main.png new file mode 100644 index 00000000000..99d429a1802 Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_hard_failed_main.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_hard_failed_settings.png b/doc/workflow/repository_mirroring/repository_mirroring_hard_failed_settings.png new file mode 100644 index 00000000000..0ab07afa3cc Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_hard_failed_settings.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_new_project.png b/doc/workflow/repository_mirroring/repository_mirroring_new_project.png new file mode 100644 index 00000000000..43bf304838f Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_new_project.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_pull_advanced_host_keys.png b/doc/workflow/repository_mirroring/repository_mirroring_pull_advanced_host_keys.png new file mode 100644 index 00000000000..5da5a7436bb Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_pull_advanced_host_keys.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_pull_settings.png b/doc/workflow/repository_mirroring/repository_mirroring_pull_settings.png new file mode 100644 index 00000000000..4b9085302a1 Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_pull_settings.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_pull_settings_for_ssh.png b/doc/workflow/repository_mirroring/repository_mirroring_pull_settings_for_ssh.png new file mode 100644 index 00000000000..8c2efdafa43 Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_pull_settings_for_ssh.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_ssh_host_keys_verified.png b/doc/workflow/repository_mirroring/repository_mirroring_ssh_host_keys_verified.png new file mode 100644 index 00000000000..93f3a532a0e Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_ssh_host_keys_verified.png differ diff --git a/doc/workflow/repository_mirroring/repository_mirroring_ssh_public_key_authentication.png b/doc/workflow/repository_mirroring/repository_mirroring_ssh_public_key_authentication.png new file mode 100644 index 00000000000..6997ad511d9 Binary files /dev/null and b/doc/workflow/repository_mirroring/repository_mirroring_ssh_public_key_authentication.png differ -- cgit v1.2.1 From b8370c9f55843351b49073dafe84a2e9858c8c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Wed, 9 May 2018 17:27:38 +0200 Subject: Support presigned multipart uploads --- app/controllers/projects/lfs_storage_controller.rb | 2 +- app/uploaders/object_storage.rb | 21 +-- .../unreleased/presigned-multipart-uploads.yml | 5 + .../artifacts_direct_upload_support.rb | 7 - config/initializers/direct_upload_support.rb | 13 ++ doc/administration/job_artifacts.md | 3 +- lib/api/runner.rb | 2 +- lib/object_storage/direct_upload.rb | 166 ++++++++++++++++++ .../artifacts_direct_upload_support_spec.rb | 71 -------- spec/initializers/direct_upload_support_spec.rb | 89 ++++++++++ spec/lib/object_storage/direct_upload_spec.rb | 188 +++++++++++++++++++++ spec/requests/api/runner_spec.rb | 1 + spec/requests/lfs_http_spec.rb | 1 + spec/support/helpers/stub_object_storage.rb | 12 ++ spec/uploaders/object_storage_spec.rb | 134 ++++++++++++--- 15 files changed, 592 insertions(+), 123 deletions(-) create mode 100644 changelogs/unreleased/presigned-multipart-uploads.yml delete mode 100644 config/initializers/artifacts_direct_upload_support.rb create mode 100644 config/initializers/direct_upload_support.rb create mode 100644 lib/object_storage/direct_upload.rb delete mode 100644 spec/initializers/artifacts_direct_upload_support_spec.rb create mode 100644 spec/initializers/direct_upload_support_spec.rb create mode 100644 spec/lib/object_storage/direct_upload_spec.rb diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb index 43d8867a536..45c98d60822 100644 --- a/app/controllers/projects/lfs_storage_controller.rb +++ b/app/controllers/projects/lfs_storage_controller.rb @@ -18,7 +18,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController def upload_authorize set_workhorse_internal_api_content_type - authorized = LfsObjectUploader.workhorse_authorize + authorized = LfsObjectUploader.workhorse_authorize(has_length: true) authorized.merge!(LfsOid: oid, LfsSize: size) render json: authorized diff --git a/app/uploaders/object_storage.rb b/app/uploaders/object_storage.rb index 5bdca26a584..3bb2e1ea63a 100644 --- a/app/uploaders/object_storage.rb +++ b/app/uploaders/object_storage.rb @@ -10,8 +10,6 @@ module ObjectStorage UnknownStoreError = Class.new(StandardError) ObjectStorageUnavailable = Class.new(StandardError) - DIRECT_UPLOAD_TIMEOUT = 4.hours - DIRECT_UPLOAD_EXPIRE_OFFSET = 15.minutes TMP_UPLOAD_PATH = 'tmp/uploads'.freeze module Store @@ -157,9 +155,9 @@ module ObjectStorage model_class.uploader_options.dig(mount_point, :mount_on) || mount_point end - def workhorse_authorize + def workhorse_authorize(has_length:, maximum_size: nil) { - RemoteObject: workhorse_remote_upload_options, + RemoteObject: workhorse_remote_upload_options(has_length: has_length, maximum_size: maximum_size), TempPath: workhorse_local_upload_path }.compact end @@ -168,23 +166,16 @@ module ObjectStorage File.join(self.root, TMP_UPLOAD_PATH) end - def workhorse_remote_upload_options + def workhorse_remote_upload_options(has_length:, maximum_size: nil) return unless self.object_store_enabled? return unless self.direct_upload_enabled? id = [CarrierWave.generate_cache_id, SecureRandom.hex].join('-') upload_path = File.join(TMP_UPLOAD_PATH, id) - connection = ::Fog::Storage.new(self.object_store_credentials) - expire_at = Time.now + DIRECT_UPLOAD_TIMEOUT + DIRECT_UPLOAD_EXPIRE_OFFSET - options = { 'Content-Type' => 'application/octet-stream' } + direct_upload = ObjectStorage::DirectUpload.new(self.object_store_credentials, remote_store_path, upload_path, + has_length: has_length, maximum_size: maximum_size) - { - ID: id, - Timeout: DIRECT_UPLOAD_TIMEOUT, - GetURL: connection.get_object_url(remote_store_path, upload_path, expire_at), - DeleteURL: connection.delete_object_url(remote_store_path, upload_path, expire_at), - StoreURL: connection.put_object_url(remote_store_path, upload_path, expire_at, options) - } + direct_upload.to_hash.merge(ID: id) end end diff --git a/changelogs/unreleased/presigned-multipart-uploads.yml b/changelogs/unreleased/presigned-multipart-uploads.yml new file mode 100644 index 00000000000..52fae6534fd --- /dev/null +++ b/changelogs/unreleased/presigned-multipart-uploads.yml @@ -0,0 +1,5 @@ +--- +title: Support direct_upload with S3 Multipart uploads +merge_request: +author: +type: added diff --git a/config/initializers/artifacts_direct_upload_support.rb b/config/initializers/artifacts_direct_upload_support.rb deleted file mode 100644 index d2bc35ea613..00000000000 --- a/config/initializers/artifacts_direct_upload_support.rb +++ /dev/null @@ -1,7 +0,0 @@ -artifacts_object_store = Gitlab.config.artifacts.object_store - -if artifacts_object_store.enabled && - artifacts_object_store.direct_upload && - artifacts_object_store.connection&.provider.to_s != 'Google' - raise "Only 'Google' is supported as a object storage provider when 'direct_upload' of artifacts is used" -end diff --git a/config/initializers/direct_upload_support.rb b/config/initializers/direct_upload_support.rb new file mode 100644 index 00000000000..8ba1229415f --- /dev/null +++ b/config/initializers/direct_upload_support.rb @@ -0,0 +1,13 @@ +SUPPORTED_DIRECT_UPLOAD_PROVIDERS = %w(Google AWS).freeze + +def verify_provider_support!(object_store) + return unless object_store.enabled + return unless object_store.direct_upload + return if SUPPORTED_DIRECT_UPLOAD_PROVIDERS.include?(object_store.connection&.provider.to_s) + + raise "Only #{SUPPORTED_DIRECT_UPLOAD_PROVIDERS.join(',')} are supported as a object storage provider when 'direct_upload' is used" +end + +verify_provider_support!(Gitlab.config.artifacts.object_store) +verify_provider_support!(Gitlab.config.uploads.object_store) +verify_provider_support!(Gitlab.config.lfs.object_store) diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 77fe4d561a1..e59ab5a72e1 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -94,6 +94,7 @@ _The artifacts are stored by default in > Available in [GitLab Premium](https://about.gitlab.com/products/) and [GitLab.com Silver](https://about.gitlab.com/gitlab-com/). > Since version 10.6, available in [GitLab CE](https://about.gitlab.com/products/) +> Since version 11.0, we support direct_upload to S3. If you don't want to use the local disk where GitLab is installed to store the artifacts, you can use an object storage like AWS S3 instead. @@ -108,7 +109,7 @@ For source installations the following settings are nested under `artifacts:` an |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where Artifacts will be stored| | -| `direct_upload` | Set to true to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. Currently only `Google` provider is supported | `false` | +| `direct_upload` | Set to true to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` | | `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | | `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | diff --git a/lib/api/runner.rb b/lib/api/runner.rb index e9886c76870..db502697a19 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -205,7 +205,7 @@ module API status 200 content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - JobArtifactUploader.workhorse_authorize + JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_artifacts_size) end desc 'Upload artifacts for job' do diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb new file mode 100644 index 00000000000..a4d6b79eb45 --- /dev/null +++ b/lib/object_storage/direct_upload.rb @@ -0,0 +1,166 @@ +module ObjectStorage + # + # The DirectUpload c;ass generates a set of presigned URLs + # that can be used to upload data to object storage from untrusted component: Workhorse, Runner? + # + # For Google it assumes that the platform supports variable Content-Length. + # + # For AWS it initiates Multipart Upload and presignes a set of part uploads. + # Class calculates the best part size to be able to upload up to asked maximum size. + # The number of generated parts will never go above 100, + # but we will always try to reduce amount of generated parts. + # The part size is rounded-up to 5MB. + # + class DirectUpload + include Gitlab::Utils::StrongMemoize + + TIMEOUT = 4.hours + EXPIRE_OFFSET = 15.minutes + + MAXIMUM_MULTIPART_PARTS = 100 + MINIMUM_MULTIPART_SIZE = 5.megabytes + + attr_reader :credentials, :bucket_name, :object_name + attr_reader :has_length, :maximum_size + + def initialize(credentials, bucket_name, object_name, has_length:, maximum_size: nil) + unless has_length + raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size + end + + @credentials = credentials + @bucket_name = bucket_name + @object_name = object_name + @has_length = has_length + @maximum_size = maximum_size + end + + def to_hash + { + Timeout: TIMEOUT, + GetURL: get_url, + StoreURL: store_url, + DeleteURL: delete_url, + MultipartUpload: multipart_upload_hash + }.compact + end + + def multipart_upload_hash + return unless requires_multipart_upload? + + { + PartSize: rounded_multipart_part_size, + PartURLs: multipart_part_urls, + CompleteURL: multipart_complete_url, + AbortURL: multipart_abort_url + } + end + + def provider + credentials[:provider].to_s + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html + def get_url + connection.get_object_url(bucket_name, object_name, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html + def delete_url + connection.delete_object_url(bucket_name, object_name, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html + def store_url + connection.put_object_url(bucket_name, object_name, expire_at, upload_options) + end + + def multipart_part_urls + Array.new(number_of_multipart_parts) do |part_index| + multipart_part_upload_url(part_index + 1) + end + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html + def multipart_part_upload_url(part_number) + connection.signed_url({ + method: 'PUT', + bucket_name: bucket_name, + object_name: object_name, + query: { uploadId: upload_id, partNumber: part_number }, + headers: upload_options + }, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html + def multipart_complete_url + connection.signed_url({ + method: 'POST', + bucket_name: bucket_name, + object_name: object_name, + query: { uploadId: upload_id }, + headers: { 'Content-Type' => 'application/xml' } + }, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadAbort.html + def multipart_abort_url + connection.signed_url({ + method: 'DELETE', + bucket_name: bucket_name, + object_name: object_name, + query: { uploadId: upload_id } + }, expire_at) + end + + private + + def rounded_multipart_part_size + # round multipart_part_size up to minimum_mulitpart_size + (multipart_part_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE * MINIMUM_MULTIPART_SIZE + end + + def multipart_part_size + maximum_size / number_of_multipart_parts + end + + def number_of_multipart_parts + [ + # round maximum_size up to minimum_mulitpart_size + (maximum_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE, + MAXIMUM_MULTIPART_PARTS + ].min + end + + def aws? + provider == 'AWS' + end + + def requires_multipart_upload? + aws? && !has_length + end + + def upload_id + return unless requires_multipart_upload? + + strong_memoize(:upload_id) do + new_upload = connection.initiate_multipart_upload(bucket_name, object_name) + new_upload.body["UploadId"] + end + end + + def expire_at + strong_memoize(:expire_at) do + Time.now + TIMEOUT + EXPIRE_OFFSET + end + end + + def upload_options + { 'Content-Type' => 'application/octet-stream' } + end + + def connection + @connection ||= ::Fog::Storage.new(credentials) + end + end +end diff --git a/spec/initializers/artifacts_direct_upload_support_spec.rb b/spec/initializers/artifacts_direct_upload_support_spec.rb deleted file mode 100644 index bfb71da3388..00000000000 --- a/spec/initializers/artifacts_direct_upload_support_spec.rb +++ /dev/null @@ -1,71 +0,0 @@ -require 'spec_helper' - -describe 'Artifacts direct upload support' do - subject do - load Rails.root.join('config/initializers/artifacts_direct_upload_support.rb') - end - - let(:connection) do - { provider: provider } - end - - before do - stub_artifacts_setting( - object_store: { - enabled: enabled, - direct_upload: direct_upload, - connection: connection - }) - end - - context 'when object storage is enabled' do - let(:enabled) { true } - - context 'when direct upload is enabled' do - let(:direct_upload) { true } - - context 'when provider is Google' do - let(:provider) { 'Google' } - - it 'succeeds' do - expect { subject }.not_to raise_error - end - end - - context 'when connection is empty' do - let(:connection) { nil } - - it 'raises an error' do - expect { subject }.to raise_error /object storage provider when 'direct_upload' of artifacts is used/ - end - end - - context 'when other provider is used' do - let(:provider) { 'AWS' } - - it 'raises an error' do - expect { subject }.to raise_error /object storage provider when 'direct_upload' of artifacts is used/ - end - end - end - - context 'when direct upload is disabled' do - let(:direct_upload) { false } - let(:provider) { 'AWS' } - - it 'succeeds' do - expect { subject }.not_to raise_error - end - end - end - - context 'when object storage is disabled' do - let(:enabled) { false } - let(:direct_upload) { false } - let(:provider) { 'AWS' } - - it 'succeeds' do - expect { subject }.not_to raise_error - end - end -end diff --git a/spec/initializers/direct_upload_support_spec.rb b/spec/initializers/direct_upload_support_spec.rb new file mode 100644 index 00000000000..f124e726bac --- /dev/null +++ b/spec/initializers/direct_upload_support_spec.rb @@ -0,0 +1,89 @@ +require 'spec_helper' + +describe 'Direct upload support' do + subject do + load Rails.root.join('config/initializers/direct_upload_support.rb') + end + + where(:config_name) do + ['lfs', 'artifacts', 'uploads'] + end + + with_them do + let(:connection) do + { provider: provider } + end + + let(:object_store) do + { + enabled: enabled, + direct_upload: direct_upload, + connection: connection + } + end + + before do + allow(Gitlab.config).to receive_messages(to_settings( + config_name => { object_store: object_store })) + end + + context 'when object storage is enabled' do + let(:enabled) { true } + + context 'when direct upload is enabled' do + let(:direct_upload) { true } + + context 'when provider is AWS' do + let(:provider) { 'AWS' } + + it 'succeeds' do + expect { subject }.not_to raise_error + end + end + + context 'when provider is Google' do + let(:provider) { 'Google' } + + it 'succeeds' do + expect { subject }.not_to raise_error + end + end + + context 'when connection is empty' do + let(:connection) { nil } + + it 'raises an error' do + expect { subject }.to raise_error /are supported as a object storage provider when 'direct_upload' is used/ + end + end + + context 'when other provider is used' do + let(:provider) { 'Rackspace' } + + it 'raises an error' do + expect { subject }.to raise_error /are supported as a object storage provider when 'direct_upload' is used/ + end + end + end + + context 'when direct upload is disabled' do + let(:direct_upload) { false } + let(:provider) { 'AWS' } + + it 'succeeds' do + expect { subject }.not_to raise_error + end + end + end + + context 'when object storage is disabled' do + let(:enabled) { false } + let(:direct_upload) { false } + let(:provider) { 'Rackspace' } + + it 'succeeds' do + expect { subject }.not_to raise_error + end + end + end +end diff --git a/spec/lib/object_storage/direct_upload_spec.rb b/spec/lib/object_storage/direct_upload_spec.rb new file mode 100644 index 00000000000..0f86d10b881 --- /dev/null +++ b/spec/lib/object_storage/direct_upload_spec.rb @@ -0,0 +1,188 @@ +require 'spec_helper' + +describe ObjectStorage::DirectUpload do + let(:credentials) do + { + provider: 'AWS', + aws_access_key_id: 'AWS_ACCESS_KEY_ID', + aws_secret_access_key: 'AWS_SECRET_ACCESS_KEY' + } + end + + let(:storage_url) { 'https://uploads.s3.amazonaws.com/' } + + let(:bucket_name) { 'uploads' } + let(:object_name) { 'tmp/uploads/my-file' } + let(:maximum_size) { 1.gigabyte } + + let(:direct_upload) { described_class.new(credentials, bucket_name, object_name, has_length: has_length, maximum_size: maximum_size) } + + describe '#has_length' do + context 'is known' do + let(:has_length) { true } + let(:maximum_size) { nil } + + it "maximum size is not required" do + expect { direct_upload }.not_to raise_error + end + end + + context 'is unknown' do + let(:has_length) { false } + + context 'and maximum size is specified' do + let(:maximum_size) { 1.gigabyte } + + it "does not raise an error" do + expect { direct_upload }.not_to raise_error + end + end + + context 'and maximum size is not specified' do + let(:maximum_size) { nil } + + it "raises an error" do + expect { direct_upload }.to raise_error /maximum_size has to be specified if length is unknown/ + end + end + end + end + + describe '#to_hash' do + subject { direct_upload.to_hash } + + shared_examples 'a valid upload' do + it "returns valid structure" do + expect(subject).to have_key(:Timeout) + expect(subject[:GetURL]).to start_with(storage_url) + expect(subject[:StoreURL]).to start_with(storage_url) + expect(subject[:DeleteURL]).to start_with(storage_url) + end + end + + shared_examples 'a valid upload with multipart data' do + before do + stub_object_storage_multipart_init(storage_url, "myUpload") + end + + it_behaves_like 'a valid upload' + + it "returns valid structure" do + expect(subject).to have_key(:MultipartUpload) + expect(subject[:MultipartUpload]).to have_key(:PartSize) + expect(subject[:MultipartUpload][:PartURLs]).to all(start_with(storage_url)) + expect(subject[:MultipartUpload][:PartURLs]).to all(include('uploadId=myUpload')) + expect(subject[:MultipartUpload][:CompleteURL]).to start_with(storage_url) + expect(subject[:MultipartUpload][:CompleteURL]).to include('uploadId=myUpload') + expect(subject[:MultipartUpload][:AbortURL]).to start_with(storage_url) + expect(subject[:MultipartUpload][:AbortURL]).to include('uploadId=myUpload') + end + end + + shared_examples 'a valid upload without multipart data' do + it_behaves_like 'a valid upload' + + it "returns valid structure" do + expect(subject).not_to have_key(:MultipartUpload) + end + end + + context 'when AWS is used' do + context 'when length is known' do + let(:has_length) { true } + + it_behaves_like 'a valid upload without multipart data' + end + + context 'when length is unknown' do + let(:has_length) { false } + + it_behaves_like 'a valid upload with multipart data' do + context 'when maximum upload size is 10MB' do + let(:maximum_size) { 10.megabyte } + + it 'returns only 2 parts' do + expect(subject[:MultipartUpload][:PartURLs].length).to eq(2) + end + + it 'part size is mimimum, 5MB' do + expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) + end + end + + context 'when maximum upload size is 12MB' do + let(:maximum_size) { 12.megabyte } + + it 'returns only 3 parts' do + expect(subject[:MultipartUpload][:PartURLs].length).to eq(3) + end + + it 'part size is rounded-up to 5MB' do + expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte) + end + end + + context 'when maximum upload size is 49GB' do + let(:maximum_size) { 49.gigabyte } + + it 'returns maximum, 100 parts' do + expect(subject[:MultipartUpload][:PartURLs].length).to eq(100) + end + + it 'part size is rounded-up to 5MB' do + expect(subject[:MultipartUpload][:PartSize]).to eq(505.megabyte) + end + end + end + end + end + + context 'when Google is used' do + let(:credentials) do + { + provider: 'Google', + google_storage_access_key_id: 'GOOGLE_ACCESS_KEY_ID', + google_storage_secret_access_key: 'GOOGLE_SECRET_ACCESS_KEY' + } + end + + let(:storage_url) { 'https://storage.googleapis.com/uploads/' } + + context 'when length is known' do + let(:has_length) { true } + + it_behaves_like 'a valid upload without multipart data' + end + + context 'when length is unknown' do + let(:has_length) { false } + + it_behaves_like 'a valid upload without multipart data' + end + end + end + + describe '#get_url' do + # this method can only be tested with integration tests + end + + describe '#delete_url' do + # this method can only be tested with integration tests + end + + describe '#store_url' do + # this method can only be tested with integration tests + end + + describe '#multipart_part_upload_url' do + # this method can only be tested with integration tests + end + + describe '#multipart_complete_url' do + # this method can only be tested with integration tests + end + + describe '#multipart_abort_url' do + # this method can only be tested with integration tests + end +end diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 319ac389083..c981a10ac38 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -1101,6 +1101,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do expect(json_response['RemoteObject']).to have_key('GetURL') expect(json_response['RemoteObject']).to have_key('StoreURL') expect(json_response['RemoteObject']).to have_key('DeleteURL') + expect(json_response['RemoteObject']).to have_key('MultipartUpload') end end diff --git a/spec/requests/lfs_http_spec.rb b/spec/requests/lfs_http_spec.rb index 79672fe1cc5..4d30b99262e 100644 --- a/spec/requests/lfs_http_spec.rb +++ b/spec/requests/lfs_http_spec.rb @@ -1021,6 +1021,7 @@ describe 'Git LFS API and storage' do expect(json_response['RemoteObject']).to have_key('GetURL') expect(json_response['RemoteObject']).to have_key('StoreURL') expect(json_response['RemoteObject']).to have_key('DeleteURL') + expect(json_response['RemoteObject']).not_to have_key('MultipartUpload') expect(json_response['LfsOid']).to eq(sample_oid) expect(json_response['LfsSize']).to eq(sample_size) end diff --git a/spec/support/helpers/stub_object_storage.rb b/spec/support/helpers/stub_object_storage.rb index 19d744b959a..80204bdab8b 100644 --- a/spec/support/helpers/stub_object_storage.rb +++ b/spec/support/helpers/stub_object_storage.rb @@ -45,4 +45,16 @@ module StubObjectStorage remote_directory: 'uploads', **params) end + + def stub_object_storage_multipart_init(endpoint, upload_id = "upload_id") + stub_request(:post, %r{\A#{endpoint}tmp/uploads/[a-z0-9-]*\?uploads\z}). + to_return status: 200, body: <<-EOS.strip_heredoc + + + example-bucket + example-object + #{upload_id} + + EOS + end end diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb index 2dd0925a8e6..01166865e88 100644 --- a/spec/uploaders/object_storage_spec.rb +++ b/spec/uploaders/object_storage_spec.rb @@ -355,7 +355,10 @@ describe ObjectStorage do end describe '.workhorse_authorize' do - subject { uploader_class.workhorse_authorize } + let(:has_length) { true } + let(:maximum_size) { nil } + + subject { uploader_class.workhorse_authorize(has_length: has_length, maximum_size: maximum_size) } before do # ensure that we use regular Fog libraries @@ -371,10 +374,6 @@ describe ObjectStorage do expect(subject[:TempPath]).to start_with(uploader_class.root) expect(subject[:TempPath]).to include(described_class::TMP_UPLOAD_PATH) end - - it "does not return remote store" do - is_expected.not_to have_key('RemoteObject') - end end shared_examples 'uses remote storage' do @@ -383,7 +382,7 @@ describe ObjectStorage do expect(subject[:RemoteObject]).to have_key(:ID) expect(subject[:RemoteObject]).to include(Timeout: a_kind_of(Integer)) - expect(subject[:RemoteObject][:Timeout]).to be(ObjectStorage::DIRECT_UPLOAD_TIMEOUT) + expect(subject[:RemoteObject][:Timeout]).to be(ObjectStorage::DirectUpload::TIMEOUT) expect(subject[:RemoteObject]).to have_key(:GetURL) expect(subject[:RemoteObject]).to have_key(:DeleteURL) expect(subject[:RemoteObject]).to have_key(:StoreURL) @@ -391,9 +390,31 @@ describe ObjectStorage do expect(subject[:RemoteObject][:DeleteURL]).to include(described_class::TMP_UPLOAD_PATH) expect(subject[:RemoteObject][:StoreURL]).to include(described_class::TMP_UPLOAD_PATH) end + end - it "does not return local store" do - is_expected.not_to have_key('TempPath') + shared_examples 'uses remote storage with multipart uploads' do + it_behaves_like 'uses remote storage' do + it "returns multipart upload" do + is_expected.to have_key(:RemoteObject) + + expect(subject[:RemoteObject]).to have_key(:MultipartUpload) + expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:PartSize) + expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:PartURLs) + expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:CompleteURL) + expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:AbortURL) + expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(include(described_class::TMP_UPLOAD_PATH)) + expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to include(described_class::TMP_UPLOAD_PATH) + expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to include(described_class::TMP_UPLOAD_PATH) + end + end + end + + shared_examples 'uses remote storage without multipart uploads' do + it_behaves_like 'uses remote storage' do + it "does not return multipart upload" do + is_expected.to have_key(:RemoteObject) + expect(subject[:RemoteObject]).not_to have_key(:MultipartUpload) + end end end @@ -416,6 +437,8 @@ describe ObjectStorage do end context 'uses AWS' do + let(:storage_url) { "https://uploads.s3-eu-central-1.amazonaws.com/" } + before do expect(uploader_class).to receive(:object_store_credentials) do { provider: "AWS", @@ -425,18 +448,40 @@ describe ObjectStorage do end end - it_behaves_like 'uses remote storage' do - let(:storage_url) { "https://uploads.s3-eu-central-1.amazonaws.com/" } + context 'for known length' do + it_behaves_like 'uses remote storage without multipart uploads' do + it 'returns links for S3' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + end + end + end + + context 'for unknown length' do + let(:has_length) { false } + let(:maximum_size) { 1.gigabyte } - it 'returns links for S3' do - expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) - expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) - expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + before do + stub_object_storage_multipart_init(storage_url) + end + + it_behaves_like 'uses remote storage with multipart uploads' do + it 'returns links for S3' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(start_with(storage_url)) + expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to start_with(storage_url) + end end end end context 'uses Google' do + let(:storage_url) { "https://storage.googleapis.com/uploads/" } + before do expect(uploader_class).to receive(:object_store_credentials) do { provider: "Google", @@ -445,36 +490,71 @@ describe ObjectStorage do end end - it_behaves_like 'uses remote storage' do - let(:storage_url) { "https://storage.googleapis.com/uploads/" } + context 'for known length' do + it_behaves_like 'uses remote storage without multipart uploads' do + it 'returns links for Google Cloud' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + end + end + end + + context 'for unknown length' do + let(:has_length) { false } + let(:maximum_size) { 1.gigabyte } - it 'returns links for Google Cloud' do - expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) - expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) - expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + it_behaves_like 'uses remote storage without multipart uploads' do + it 'returns links for Google Cloud' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + end end end end context 'uses GDK/minio' do + let(:storage_url) { "http://minio:9000/uploads/" } + before do expect(uploader_class).to receive(:object_store_credentials) do { provider: "AWS", aws_access_key_id: "AWS_ACCESS_KEY_ID", aws_secret_access_key: "AWS_SECRET_ACCESS_KEY", - endpoint: 'http://127.0.0.1:9000', + endpoint: 'http://minio:9000', path_style: true, region: "gdk" } end end - it_behaves_like 'uses remote storage' do - let(:storage_url) { "http://127.0.0.1:9000/uploads/" } + context 'for known length' do + it_behaves_like 'uses remote storage without multipart uploads' do + it 'returns links for S3' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + end + end + end + + context 'for unknown length' do + let(:has_length) { false } + let(:maximum_size) { 1.gigabyte } - it 'returns links for S3' do - expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) - expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) - expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + before do + stub_object_storage_multipart_init(storage_url) + end + + it_behaves_like 'uses remote storage with multipart uploads' do + it 'returns links for S3' do + expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(start_with(storage_url)) + expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to start_with(storage_url) + expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to start_with(storage_url) + end end end end -- cgit v1.2.1 From 7350eb1fa83662d4aaa7541acb387b3742ba9788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Mon, 4 Jun 2018 11:41:37 +0000 Subject: Add ability to search wiki titles --- app/helpers/search_helper.rb | 12 +++- app/models/project_wiki.rb | 4 -- app/views/search/results/_blob.html.haml | 18 ++--- app/views/search/results/_blob_data.html.haml | 9 +++ app/views/search/results/_wiki_blob.html.haml | 15 ++-- .../fj-34526-enabling-wiki-search-by-title.yml | 5 ++ lib/api/search.rb | 4 +- lib/gitlab/file_finder.rb | 26 +++++-- lib/gitlab/project_search_results.rb | 3 +- lib/gitlab/wiki_file_finder.rb | 23 ++++++ spec/lib/gitlab/file_finder_spec.rb | 24 ++----- spec/lib/gitlab/project_search_results_spec.rb | 82 +++++++++------------- spec/lib/gitlab/wiki_file_finder_spec.rb | 20 ++++++ spec/support/shared_examples/file_finder.rb | 21 ++++++ 14 files changed, 159 insertions(+), 107 deletions(-) create mode 100644 app/views/search/results/_blob_data.html.haml create mode 100644 changelogs/unreleased/fj-34526-enabling-wiki-search-by-title.yml create mode 100644 lib/gitlab/wiki_file_finder.rb create mode 100644 spec/lib/gitlab/wiki_file_finder_spec.rb create mode 100644 spec/support/shared_examples/file_finder.rb diff --git a/app/helpers/search_helper.rb b/app/helpers/search_helper.rb index 761c1252fc8..f7dafca7834 100644 --- a/app/helpers/search_helper.rb +++ b/app/helpers/search_helper.rb @@ -25,14 +25,22 @@ module SearchHelper return unless collection.count > 0 from = collection.offset_value + 1 - to = collection.offset_value + collection.length + to = collection.offset_value + collection.count count = collection.total_count "Showing #{from} - #{to} of #{count} #{scope.humanize(capitalize: false)} for \"#{term}\"" end + def find_project_for_result_blob(result) + @project + end + def parse_search_result(result) - Gitlab::ProjectSearchResults.parse_search_result(result) + result + end + + def search_blob_title(project, filename) + filename end private diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index f799a0b4227..a6f94b3e3b0 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -140,10 +140,6 @@ class ProjectWiki [title, title_array.join("/")] end - def search_files(query) - repository.search_files_by_content(query, default_branch) - end - def repository @repository ||= Repository.new(full_path, @project, disk_path: disk_path, is_wiki: true) end diff --git a/app/views/search/results/_blob.html.haml b/app/views/search/results/_blob.html.haml index de473c23d66..fdcd126e7a3 100644 --- a/app/views/search/results/_blob.html.haml +++ b/app/views/search/results/_blob.html.haml @@ -1,13 +1,5 @@ -- file_name, blob = blob -.blob-result - .file-holder - .js-file-title.file-title - - ref = @search_results.repository_ref - - blob_link = project_blob_path(@project, tree_join(ref, file_name)) - = link_to blob_link do - %i.fa.fa-file - %strong - = file_name - - if blob - .file-content.code.term - = render 'shared/file_highlight', blob: blob, first_line_number: blob.startline, blob_link: blob_link +- project = find_project_for_result_blob(blob) +- file_name, blob = parse_search_result(blob) +- blob_link = project_blob_path(project, tree_join(blob.ref, file_name)) + += render partial: 'search/results/blob_data', locals: { blob: blob, project: project, file_name: file_name, blob_link: blob_link } diff --git a/app/views/search/results/_blob_data.html.haml b/app/views/search/results/_blob_data.html.haml new file mode 100644 index 00000000000..0115be41ff1 --- /dev/null +++ b/app/views/search/results/_blob_data.html.haml @@ -0,0 +1,9 @@ +.blob-result + .file-holder + .js-file-title.file-title + = link_to blob_link do + %i.fa.fa-file + = search_blob_title(project, file_name) + - if blob.data + .file-content.code.term + = render 'shared/file_highlight', blob: blob, first_line_number: blob.startline diff --git a/app/views/search/results/_wiki_blob.html.haml b/app/views/search/results/_wiki_blob.html.haml index 16a0e432d62..4346217c230 100644 --- a/app/views/search/results/_wiki_blob.html.haml +++ b/app/views/search/results/_wiki_blob.html.haml @@ -1,10 +1,5 @@ -- wiki_blob = parse_search_result(wiki_blob) -.blob-result - .file-holder - .js-file-title.file-title - = link_to project_wiki_path(@project, wiki_blob.basename) do - %i.fa.fa-file - %strong - = wiki_blob.basename - .file-content.code.term - = render 'shared/file_highlight', blob: wiki_blob, first_line_number: wiki_blob.startline +- project = find_project_for_result_blob(wiki_blob) +- file_name, wiki_blob = parse_search_result(wiki_blob) +- wiki_blob_link = project_wiki_path(project, wiki_blob.basename) + += render partial: 'search/results/blob_data', locals: { blob: wiki_blob, project: project, file_name: file_name, blob_link: wiki_blob_link } diff --git a/changelogs/unreleased/fj-34526-enabling-wiki-search-by-title.yml b/changelogs/unreleased/fj-34526-enabling-wiki-search-by-title.yml new file mode 100644 index 00000000000..2ae2cf8a23e --- /dev/null +++ b/changelogs/unreleased/fj-34526-enabling-wiki-search-by-title.yml @@ -0,0 +1,5 @@ +--- +title: Added ability to search by wiki titles +merge_request: 19112 +author: +type: added diff --git a/lib/api/search.rb b/lib/api/search.rb index 5d9ec617cb7..37fbabe419c 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -34,9 +34,7 @@ module API def process_results(results) case params[:scope] - when 'wiki_blobs' - paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob, user_project) } - when 'blobs' + when 'blobs', 'wiki_blobs' paginate(results).map { |blob| blob[1] } else paginate(results) diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 8c082c0c336..f42088f980e 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -32,17 +32,13 @@ module Gitlab end def find_by_filename(query, except: []) - filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) - filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + filenames = search_filenames(query, except) - blob_refs = filenames.map { |filename| [ref, filename] } - blobs = Gitlab::Git::Blob.batch(repository, blob_refs, blob_size_limit: 1024) - - blobs.map do |blob| + blobs(filenames).map do |blob| Gitlab::SearchResults::FoundBlob.new( id: blob.id, filename: blob.path, - basename: File.basename(blob.path), + basename: File.basename(blob.path, File.extname(blob.path)), ref: ref, startline: 1, data: blob.data, @@ -50,5 +46,21 @@ module Gitlab ) end end + + def search_filenames(query, except) + filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) + + filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + + filenames + end + + def blob_refs(filenames) + filenames.map { |filename| [ref, filename] } + end + + def blobs(filenames) + Gitlab::Git::Blob.batch(repository, blob_refs(filenames), blob_size_limit: 1024) + end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 2e9b6e302f5..38bdc61d8ab 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -106,7 +106,8 @@ module Gitlab project_wiki = ProjectWiki.new(project) unless project_wiki.empty? - project_wiki.search_files(query) + ref = repository_ref || project.wiki.default_branch + Gitlab::WikiFileFinder.new(project, ref).find(query) else [] end diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb new file mode 100644 index 00000000000..f97278f05cd --- /dev/null +++ b/lib/gitlab/wiki_file_finder.rb @@ -0,0 +1,23 @@ +module Gitlab + class WikiFileFinder < FileFinder + attr_reader :repository + + def initialize(project, ref) + @project = project + @ref = ref + @repository = project.wiki.repository + end + + private + + def search_filenames(query, except) + safe_query = Regexp.escape(query.tr(' ', '-')) + safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) + filenames = repository.ls_files(ref) + + filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + + filenames.grep(safe_query).first(BATCH_SIZE) + end + end +end diff --git a/spec/lib/gitlab/file_finder_spec.rb b/spec/lib/gitlab/file_finder_spec.rb index 07cb10e563e..d6d9e4001a3 100644 --- a/spec/lib/gitlab/file_finder_spec.rb +++ b/spec/lib/gitlab/file_finder_spec.rb @@ -3,27 +3,11 @@ require 'spec_helper' describe Gitlab::FileFinder do describe '#find' do let(:project) { create(:project, :public, :repository) } - let(:finder) { described_class.new(project, project.default_branch) } - it 'finds by name' do - results = finder.find('files') - - filename, blob = results.find { |_, blob| blob.filename == 'files/images/wm.svg' } - expect(filename).to eq('files/images/wm.svg') - expect(blob).to be_a(Gitlab::SearchResults::FoundBlob) - expect(blob.ref).to eq(finder.ref) - expect(blob.data).not_to be_empty - end - - it 'finds by content' do - results = finder.find('files') - - filename, blob = results.find { |_, blob| blob.filename == 'CHANGELOG' } - - expect(filename).to eq('CHANGELOG') - expect(blob).to be_a(Gitlab::SearchResults::FoundBlob) - expect(blob.ref).to eq(finder.ref) - expect(blob.data).not_to be_empty + it_behaves_like 'file finder' do + subject { described_class.new(project, project.default_branch) } + let(:expected_file_by_name) { 'files/images/wm.svg' } + let(:expected_file_by_content) { 'CHANGELOG' } end end end diff --git a/spec/lib/gitlab/project_search_results_spec.rb b/spec/lib/gitlab/project_search_results_spec.rb index e3f705d2299..50224bde722 100644 --- a/spec/lib/gitlab/project_search_results_spec.rb +++ b/spec/lib/gitlab/project_search_results_spec.rb @@ -22,47 +22,57 @@ describe Gitlab::ProjectSearchResults do it { expect(results.query).to eq('hello world') } end - describe 'blob search' do - let(:project) { create(:project, :public, :repository) } - - subject(:results) { described_class.new(user, project, 'files').objects('blobs') } - - context 'when repository is disabled' do - let(:project) { create(:project, :public, :repository, :repository_disabled) } + shared_examples 'general blob search' do |entity_type, blob_kind| + let(:query) { 'files' } + subject(:results) { described_class.new(user, project, query).objects(blob_type) } - it 'hides blobs from members' do + context "when #{entity_type} is disabled" do + let(:project) { disabled_project } + it "hides #{blob_kind} from members" do project.add_reporter(user) is_expected.to be_empty end - it 'hides blobs from non-members' do + it "hides #{blob_kind} from non-members" do is_expected.to be_empty end end - context 'when repository is internal' do - let(:project) { create(:project, :public, :repository, :repository_private) } + context "when #{entity_type} is internal" do + let(:project) { private_project } - it 'finds blobs for members' do + it "finds #{blob_kind} for members" do project.add_reporter(user) is_expected.not_to be_empty end - it 'hides blobs from non-members' do + it "hides #{blob_kind} from non-members" do is_expected.to be_empty end end it 'finds by name' do - expect(results.map(&:first)).to include('files/images/wm.svg') + expect(results.map(&:first)).to include(expected_file_by_name) end it 'finds by content' do - blob = results.select { |result| result.first == "CHANGELOG" }.flatten.last + blob = results.select { |result| result.first == expected_file_by_content }.flatten.last - expect(blob.filename).to eq("CHANGELOG") + expect(blob.filename).to eq(expected_file_by_content) + end + end + + describe 'blob search' do + let(:project) { create(:project, :public, :repository) } + + it_behaves_like 'general blob search', 'repository', 'blobs' do + let(:blob_type) { 'blobs' } + let(:disabled_project) { create(:project, :public, :repository, :repository_disabled) } + let(:private_project) { create(:project, :public, :repository, :repository_private) } + let(:expected_file_by_name) { 'files/images/wm.svg' } + let(:expected_file_by_content) { 'CHANGELOG' } end describe 'parsing results' do @@ -189,40 +199,18 @@ describe Gitlab::ProjectSearchResults do describe 'wiki search' do let(:project) { create(:project, :public, :wiki_repo) } let(:wiki) { build(:project_wiki, project: project) } - let!(:wiki_page) { wiki.create_page('Title', 'Content') } - - subject(:results) { described_class.new(user, project, 'Content').objects('wiki_blobs') } - - context 'when wiki is disabled' do - let(:project) { create(:project, :public, :wiki_repo, :wiki_disabled) } - it 'hides wiki blobs from members' do - project.add_reporter(user) - - is_expected.to be_empty - end - - it 'hides wiki blobs from non-members' do - is_expected.to be_empty - end - end - - context 'when wiki is internal' do - let(:project) { create(:project, :public, :wiki_repo, :wiki_private) } - - it 'finds wiki blobs for guest' do - project.add_guest(user) - - is_expected.not_to be_empty - end - - it 'hides wiki blobs from non-members' do - is_expected.to be_empty - end + before do + wiki.create_page('Files/Title', 'Content') + wiki.create_page('CHANGELOG', 'Files example') end - it 'finds by content' do - expect(results).to include("master:Title.md\x001\x00Content\n") + it_behaves_like 'general blob search', 'wiki', 'wiki blobs' do + let(:blob_type) { 'wiki_blobs' } + let(:disabled_project) { create(:project, :public, :wiki_repo, :wiki_disabled) } + let(:private_project) { create(:project, :public, :wiki_repo, :wiki_private) } + let(:expected_file_by_name) { 'Files/Title.md' } + let(:expected_file_by_content) { 'CHANGELOG.md' } end end diff --git a/spec/lib/gitlab/wiki_file_finder_spec.rb b/spec/lib/gitlab/wiki_file_finder_spec.rb new file mode 100644 index 00000000000..025d1203dc5 --- /dev/null +++ b/spec/lib/gitlab/wiki_file_finder_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +describe Gitlab::WikiFileFinder do + describe '#find' do + let(:project) { create(:project, :public, :wiki_repo) } + let(:wiki) { build(:project_wiki, project: project) } + + before do + wiki.create_page('Files/Title', 'Content') + wiki.create_page('CHANGELOG', 'Files example') + end + + it_behaves_like 'file finder' do + subject { described_class.new(project, project.wiki.default_branch) } + + let(:expected_file_by_name) { 'Files/Title.md' } + let(:expected_file_by_content) { 'CHANGELOG.md' } + end + end +end diff --git a/spec/support/shared_examples/file_finder.rb b/spec/support/shared_examples/file_finder.rb new file mode 100644 index 00000000000..ef144bdf61c --- /dev/null +++ b/spec/support/shared_examples/file_finder.rb @@ -0,0 +1,21 @@ +shared_examples 'file finder' do + let(:query) { 'files' } + let(:search_results) { subject.find(query) } + + it 'finds by name' do + filename, blob = search_results.find { |_, blob| blob.filename == expected_file_by_name } + expect(filename).to eq(expected_file_by_name) + expect(blob).to be_a(Gitlab::SearchResults::FoundBlob) + expect(blob.ref).to eq(subject.ref) + expect(blob.data).not_to be_empty + end + + it 'finds by content' do + filename, blob = search_results.find { |_, blob| blob.filename == expected_file_by_content } + + expect(filename).to eq(expected_file_by_content) + expect(blob).to be_a(Gitlab::SearchResults::FoundBlob) + expect(blob.ref).to eq(subject.ref) + expect(blob.data).not_to be_empty + end +end -- cgit v1.2.1 From 4f20bf956a179bc4ffdfa23168925c102eb2e88b Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Mon, 4 Jun 2018 12:42:02 +0000 Subject: Backport EE SlashCommand Refactor --- lib/gitlab/slash_commands/command.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index bb778f37096..c82320a6036 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -1,13 +1,15 @@ module Gitlab module SlashCommands class Command < BaseCommand - COMMANDS = [ - Gitlab::SlashCommands::IssueShow, - Gitlab::SlashCommands::IssueNew, - Gitlab::SlashCommands::IssueSearch, - Gitlab::SlashCommands::IssueMove, - Gitlab::SlashCommands::Deploy - ].freeze + def self.commands + [ + Gitlab::SlashCommands::IssueShow, + Gitlab::SlashCommands::IssueNew, + Gitlab::SlashCommands::IssueSearch, + Gitlab::SlashCommands::IssueMove, + Gitlab::SlashCommands::Deploy + ] + end def execute command, match = match_command @@ -37,7 +39,7 @@ module Gitlab private def available_commands - COMMANDS.select do |klass| + self.class.commands.keep_if do |klass| klass.available?(project) end end -- cgit v1.2.1 From 56d70009b1e6fb55870c132209b152bad2a7f086 Mon Sep 17 00:00:00 2001 From: Filipa Lacerda Date: Mon, 4 Jun 2018 15:52:21 +0100 Subject: Make error in clusters occupy full width --- app/assets/javascripts/clusters/components/application_row.vue | 2 +- app/assets/stylesheets/pages/clusters.scss | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/clusters/components/application_row.vue b/app/assets/javascripts/clusters/components/application_row.vue index 30567993322..98c0b9c22a8 100644 --- a/app/assets/javascripts/clusters/components/application_row.vue +++ b/app/assets/javascripts/clusters/components/application_row.vue @@ -187,7 +187,7 @@ role="row" >
diff --git a/app/assets/stylesheets/pages/clusters.scss b/app/assets/stylesheets/pages/clusters.scss index 3e4d123242c..56beb7718a4 100644 --- a/app/assets/stylesheets/pages/clusters.scss +++ b/app/assets/stylesheets/pages/clusters.scss @@ -13,6 +13,10 @@ max-width: 100%; } +.clusters-error-alert { + width: 100%; +} + .clusters-container { .nav-bar-right { padding: $gl-padding-top $gl-padding; -- cgit v1.2.1 From 840ce49d4b682e6f39cd22804be25d3c0311969b Mon Sep 17 00:00:00 2001 From: Tim Zallmann Date: Mon, 4 Jun 2018 17:02:49 +0200 Subject: Fixes the Terms Page --- app/assets/stylesheets/framework/terms.scss | 8 +++++--- app/views/layouts/terms.html.haml | 6 +++--- app/views/users/terms/index.html.haml | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app/assets/stylesheets/framework/terms.scss b/app/assets/stylesheets/framework/terms.scss index 744fd0ff796..7cda674e5c8 100644 --- a/app/assets/stylesheets/framework/terms.scss +++ b/app/assets/stylesheets/framework/terms.scss @@ -11,15 +11,15 @@ padding-top: $gl-padding; } - .panel { - .panel-heading { + .card { + .card-header { display: -webkit-flex; display: flex; align-items: center; justify-content: space-between; line-height: $line-height-base; - .title { + .card-title { display: flex; align-items: center; @@ -34,6 +34,8 @@ .navbar-collapse { padding-right: 0; + flex-grow: 0; + flex-basis: auto; .navbar-nav { margin: 0; diff --git a/app/views/layouts/terms.html.haml b/app/views/layouts/terms.html.haml index a8964b19ba1..977eb350365 100644 --- a/app/views/layouts/terms.html.haml +++ b/app/views/layouts/terms.html.haml @@ -14,9 +14,9 @@ %div{ class: "#{container_class} limit-container-width" } .content{ id: "content-body" } - .panel.panel-default - .panel-heading - .title + .card + .card-header + .card-title = brand_header_logo - logo_text = brand_header_logo_type - if logo_text.present? diff --git a/app/views/users/terms/index.html.haml b/app/views/users/terms/index.html.haml index e0fe551cf36..b9f25a71170 100644 --- a/app/views/users/terms/index.html.haml +++ b/app/views/users/terms/index.html.haml @@ -1,8 +1,8 @@ - redirect_params = { redirect: @redirect } if @redirect -.panel-content.rendered-terms +.card-body.rendered-terms = markdown_field(@term, :terms) -.row-content-block.footer-block.clearfix +.card-footer.footer-block.clearfix - if can?(current_user, :accept_terms, @term) .float-right = button_to accept_term_path(@term, redirect_params), class: 'btn btn-success prepend-left-8' do -- cgit v1.2.1 From 2f50b206f2921faf47637af526d810bc10ffb3ef Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 4 Jun 2018 10:41:56 +0200 Subject: Hide archived projects from `shared_projects` Since we don't show the archived projects, we shouldnot load them and pass them to the fronted to be filtered out again. --- app/controllers/groups/shared_projects_controller.rb | 4 +++- spec/controllers/groups/shared_projects_controller_spec.rb | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/controllers/groups/shared_projects_controller.rb b/app/controllers/groups/shared_projects_controller.rb index f2f835767e0..7dec1f5f402 100644 --- a/app/controllers/groups/shared_projects_controller.rb +++ b/app/controllers/groups/shared_projects_controller.rb @@ -24,7 +24,9 @@ module Groups # Make the `search` param consistent for the frontend, # which will be using `filter`. params[:search] ||= params[:filter] if params[:filter] - params.permit(:sort, :search) + # Don't show archived projects + params[:non_archived] = true + params.permit(:sort, :search, :non_archived) end end end diff --git a/spec/controllers/groups/shared_projects_controller_spec.rb b/spec/controllers/groups/shared_projects_controller_spec.rb index d8fa41abb18..003c8c262e7 100644 --- a/spec/controllers/groups/shared_projects_controller_spec.rb +++ b/spec/controllers/groups/shared_projects_controller_spec.rb @@ -38,7 +38,7 @@ describe Groups::SharedProjectsController do end it 'allows filtering shared projects' do - project = create(:project, :archived, namespace: user.namespace, name: "Searching for") + project = create(:project, namespace: user.namespace, name: "Searching for") share_project(project) get_shared_projects(filter: 'search') @@ -55,5 +55,14 @@ describe Groups::SharedProjectsController do expect(json_project_ids).to eq([second_project.id, shared_project.id]) end + + it 'does not include archived projects' do + archived_project = create(:project, :archived, namespace: user.namespace) + share_project(archived_project) + + get_shared_projects + + expect(json_project_ids).to contain_exactly(shared_project.id) + end end end -- cgit v1.2.1 From fb71c46805e14419897912f6338f635a21d1d097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarka=20Kadlecov=C3=A1?= Date: Mon, 4 Jun 2018 10:38:18 +0200 Subject: Fix repository_storage spec for Rails5 --- spec/models/application_setting_spec.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 968267a6d24..3e6656e0f12 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -110,10 +110,9 @@ describe ApplicationSetting do # Upgraded databases will have this sort of content context 'repository_storages is a String, not an Array' do before do - setting.__send__(:raw_write_attribute, :repository_storages, 'default') + described_class.where(id: setting.id).update_all(repository_storages: 'default') end - it { expect(setting.repository_storages_before_type_cast).to eq('default') } it { expect(setting.repository_storages).to eq(['default']) } end -- cgit v1.2.1 From 4e322d0778ddce02749de8973a4fe0b3b4dfec77 Mon Sep 17 00:00:00 2001 From: Phil Hughes Date: Thu, 31 May 2018 11:18:21 +0100 Subject: Fixed alignment issues with IDE sidebar Fixed sidebar tooltips not hiding after clicking --- app/assets/javascripts/ide/components/activity_bar.vue | 14 +++++++++++--- app/assets/stylesheets/pages/repo.scss | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/app/assets/javascripts/ide/components/activity_bar.vue b/app/assets/javascripts/ide/components/activity_bar.vue index 05dbc1410de..6efcad6adea 100644 --- a/app/assets/javascripts/ide/components/activity_bar.vue +++ b/app/assets/javascripts/ide/components/activity_bar.vue @@ -1,4 +1,5 @@