From b21ee2ee36e1aaddbe0b3541a8cac5f117143b66 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 11:29:04 +0200 Subject: Add initial stage_id background migration files --- .../migrate_build_stage_id_reference.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 lib/gitlab/background_migration/migrate_build_stage_id_reference.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb new file mode 100644 index 00000000000..b554c3e079b --- /dev/null +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -0,0 +1,16 @@ +module Gitlab + module BackgroundMigration + class MigrateBuildStageIdReference + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + end + + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + end + + def perform(id) + end + end + end +end -- cgit v1.2.1 From 98992c4e4b3231a99eb5ff17c44e96fe79a6cff2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:01:52 +0200 Subject: Add initial build stage_id ref background migration --- .../migrate_build_stage_id_reference.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index b554c3e079b..87c6c4ed49f 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -1,15 +1,19 @@ module Gitlab module BackgroundMigration class MigrateBuildStageIdReference - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - end + def perform(id) + raise ArgumentError unless id.is_a?(Integer) - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - end + sql = <<-SQL.strip_heredoc + UPDATE "ci_builds" SET "stage_id" = ( + SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage + ) + WHERE "ci_builds"."id" = #{id} AND "ci_builds"."stage_id" IS NULL + SQL - def perform(id) + ActiveRecord::Base.connection.execute(sql) end end end -- cgit v1.2.1 From efa3511e0495f73b06e96e60538f8e719ffe97a4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 15:20:26 +0200 Subject: Test if argument passed to a migration is present --- lib/gitlab/background_migration/migrate_build_stage_id_reference.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index 87c6c4ed49f..711126ea0d3 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -2,7 +2,7 @@ module Gitlab module BackgroundMigration class MigrateBuildStageIdReference def perform(id) - raise ArgumentError unless id.is_a?(Integer) + raise ArgumentError unless id.present? sql = <<-SQL.strip_heredoc UPDATE "ci_builds" SET "stage_id" = ( -- cgit v1.2.1 From 6db8253cb86f44a80282706cc3de3df954661434 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 15:26:47 +0200 Subject: Improve readability of build stage id migration query --- .../migrate_build_stage_id_reference.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index 711126ea0d3..c8669ca3272 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -5,12 +5,13 @@ module Gitlab raise ArgumentError unless id.present? sql = <<-SQL.strip_heredoc - UPDATE "ci_builds" SET "stage_id" = ( - SELECT id FROM ci_stages - WHERE ci_stages.pipeline_id = ci_builds.commit_id - AND ci_stages.name = ci_builds.stage - ) - WHERE "ci_builds"."id" = #{id} AND "ci_builds"."stage_id" IS NULL + UPDATE "ci_builds" + SET "stage_id" = + (SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage) + WHERE "ci_builds"."id" = #{id} + AND "ci_builds"."stage_id" IS NULL SQL ActiveRecord::Base.connection.execute(sql) -- cgit v1.2.1 From 6997dfa3426b26f7eb8f294b261827ef0b6d823b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 12:07:31 +0200 Subject: Sanitize id value passed to async background migration --- lib/gitlab/background_migration/migrate_build_stage_id_reference.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index c8669ca3272..d1d0a968588 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -2,15 +2,13 @@ module Gitlab module BackgroundMigration class MigrateBuildStageIdReference def perform(id) - raise ArgumentError unless id.present? - sql = <<-SQL.strip_heredoc UPDATE "ci_builds" SET "stage_id" = (SELECT id FROM ci_stages WHERE ci_stages.pipeline_id = ci_builds.commit_id AND ci_stages.name = ci_builds.stage) - WHERE "ci_builds"."id" = #{id} + WHERE "ci_builds"."id" = #{id.to_i} AND "ci_builds"."stage_id" IS NULL SQL -- cgit v1.2.1 From 6c477d5b9496829eb5cb56ef32a0dd813be7dc16 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Jul 2017 10:54:48 +0200 Subject: Move stages status migration to the background worker --- .../background_migration/migrate_stage_status.rb | 76 ++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 lib/gitlab/background_migration/migrate_stage_status.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb new file mode 100644 index 00000000000..e4fdc723b13 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -0,0 +1,76 @@ +module Gitlab + module BackgroundMigration + class MigrateStageStatus + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 } + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + scope :latest, -> { where(retried: [false, nil]) } + scope :created, -> { where(status: 'created') } + scope :running, -> { where(status: 'running') } + scope :pending, -> { where(status: 'pending') } + scope :success, -> { where(status: 'success') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + where("allow_failure = ? OR status IN (?)", + false, %w[created pending running success skipped]) + end + + def self.status_sql + scope_relevant = latest.exclude_ignored + scope_warnings = latest.failed_but_allowed + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql + + <<-SQL.strip_heredoc + (CASE + WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} + WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} + WHEN (#{created}) > 0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} + END) + SQL + end + end + + def perform(id) + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql + + sql = <<-SQL + UPDATE ci_stages SET status = (#{status_sql}) + WHERE ci_stages.id = #{id.to_i} + SQL + + ActiveRecord::Base.connection.execute(sql) + end + end + end +end -- cgit v1.2.1 From 7082530d555ad98fede2823d2123622abaf1c3a3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 10 Jul 2017 15:42:19 +0200 Subject: Schedule stages statuses bg migrations in batches --- lib/gitlab/background_migration/migrate_stage_status.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index e4fdc723b13..3c9744d1607 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -58,7 +58,7 @@ module Gitlab end end - def perform(id) + def perform(start_id, stop_id) status_sql = Build .where('ci_builds.commit_id = ci_stages.pipeline_id') .where('ci_builds.stage = ci_stages.name') @@ -66,7 +66,8 @@ module Gitlab sql = <<-SQL UPDATE ci_stages SET status = (#{status_sql}) - WHERE ci_stages.id = #{id.to_i} + WHERE ci_stages.status IS NULL + AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} SQL ActiveRecord::Base.connection.execute(sql) -- cgit v1.2.1 From 1d087e073660130c81bf0917a6fa395886b6e2dc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 11 Jul 2017 12:01:35 +0200 Subject: Freeze mutable constants in stages migration code --- lib/gitlab/background_migration/migrate_stage_status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index 3c9744d1607..b1ff0900709 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -2,7 +2,7 @@ module Gitlab module BackgroundMigration class MigrateStageStatus STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 } + failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze class Build < ActiveRecord::Base self.table_name = 'ci_builds' -- cgit v1.2.1 From 76c3d2d434d3c550c3de912abc0a5b1dc1455368 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 9 Jun 2017 16:24:54 -0500 Subject: Add full JSON endpoints for issue notes and discussions --- lib/api/entities.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 09a88869063..89ab2118dd5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,11 +1,11 @@ module API module Entities class UserSafe < Grape::Entity - expose :name, :username + expose :id, :name, :username end class UserBasic < UserSafe - expose :id, :state + expose :state expose :avatar_url do |user, options| user.avatar_url(only_path: false) end -- cgit v1.2.1 From c21ae07e331ca14605410555d0582f14cb661bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Tue, 25 Jul 2017 16:48:17 -0400 Subject: Refactor Gitlab::Git::Commit to include a repository --- lib/gitlab/cycle_analytics/plan_event_fetcher.rb | 2 +- lib/gitlab/git/blame.rb | 2 +- lib/gitlab/git/commit.rb | 26 +++++++++--------------- lib/gitlab/git/diff_collection.rb | 3 +-- lib/gitlab/git/repository.rb | 2 +- lib/gitlab/gitaly_client/commit.rb | 14 ------------- lib/gitlab/gitaly_client/commit_service.rb | 8 +++----- lib/gitlab/gitaly_client/ref_service.rb | 8 +++----- 8 files changed, 20 insertions(+), 45 deletions(-) delete mode 100644 lib/gitlab/gitaly_client/commit.rb (limited to 'lib') diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index b260822788d..2479b4a7706 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -54,7 +54,7 @@ module Gitlab end def serialize_commit(event, commit, query) - commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project) + commit = Commit.from_hash(commit.to_hash, @project) AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 8dbe25e55f6..31effdba292 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -16,7 +16,7 @@ module Gitlab def each @blames.each do |blame| yield( - Gitlab::Git::Commit.new(blame.commit), + Gitlab::Git::Commit.new(@repo, blame.commit), blame.line ) end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 600d886e818..b08d7e8fec3 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -51,7 +51,7 @@ module Gitlab # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") return commit_id if commit_id.is_a?(Gitlab::Git::Commit) - return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) + return decorate(repo, commit_id) if commit_id.is_a?(Rugged::Commit) obj = if commit_id.is_a?(String) repo.rev_parse_target(commit_id) @@ -61,7 +61,7 @@ module Gitlab return nil unless obj.is_a?(Rugged::Commit) - decorate(obj) + decorate(repo, obj) rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository nil end @@ -102,7 +102,7 @@ module Gitlab if is_enabled repo.gitaly_commit_client.between(base, head) else - repo.rugged_commits_between(base, head).map { |c| decorate(c) } + repo.rugged_commits_between(base, head).map { |c| decorate(repo, c) } end end rescue Rugged::ReferenceError @@ -169,7 +169,7 @@ module Gitlab offset = actual_options[:skip] limit = actual_options[:max_count] walker.each(offset: offset, limit: limit) do |commit| - commits.push(decorate(commit)) + commits.push(decorate(repo, commit)) end walker.reset @@ -183,8 +183,8 @@ module Gitlab Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end - def decorate(commit, ref = nil) - Gitlab::Git::Commit.new(commit, ref) + def decorate(repository, commit, ref = nil) + Gitlab::Git::Commit.new(repository, commit, ref) end # Returns a diff object for the changes introduced by +rugged_commit+. @@ -221,7 +221,7 @@ module Gitlab end end - def initialize(raw_commit, head = nil) + def initialize(repository, raw_commit, head = nil) raise "Nil as raw commit passed" unless raw_commit case raw_commit @@ -229,12 +229,13 @@ module Gitlab init_from_hash(raw_commit) when Rugged::Commit init_from_rugged(raw_commit) - when Gitlab::GitalyClient::Commit + when Gitaly::GitCommit init_from_gitaly(raw_commit) else raise "Invalid raw commit type: #{raw_commit.class}" end + @repository = repository @head = head end @@ -309,14 +310,7 @@ module Gitlab end def parents - case raw_commit - when Rugged::Commit - raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } - when Gitlab::GitalyClient::Commit - parent_ids.map { |oid| self.class.find(raw_commit.repository, oid) }.compact - else - raise NotImplementedError, "commit source doesn't support #parents" - end + parent_ids.map { |oid| self.class.find(@repository, oid) }.compact end # Get the gpg signature of this commit. diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 87ed9c3ea26..6a601561c2a 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -28,7 +28,6 @@ module Gitlab @limits = self.class.collection_limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) - @from_gitaly = options.fetch(:from_gitaly, false) @line_count = 0 @byte_count = 0 @@ -44,7 +43,7 @@ module Gitlab return if @iterator.nil? Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled && @from_gitaly + if is_enabled && @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher) each_gitaly_patch(&block) else each_rugged_patch(&block) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f6f9d49bf37..4162526be2b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -314,7 +314,7 @@ module Gitlab options[:limit] ||= 0 options[:offset] ||= 0 - raw_log(options).map { |c| Commit.decorate(c) } + raw_log(options).map { |c| Commit.decorate(self, c) } end def count_commits(options) diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb deleted file mode 100644 index 61fe462d762..00000000000 --- a/lib/gitlab/gitaly_client/commit.rb +++ /dev/null @@ -1,14 +0,0 @@ -module Gitlab - module GitalyClient - class Commit - attr_reader :repository, :gitaly_commit - - delegate :id, :subject, :body, :author, :committer, :parent_ids, to: :gitaly_commit - - def initialize(repository, gitaly_commit) - @repository = repository - @gitaly_commit = gitaly_commit - end - end - end -end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ac6817e6d0e..2a97a025e58 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -107,8 +107,7 @@ module Gitlab gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit return unless gitaly_commit - commit = GitalyClient::Commit.new(@repository, gitaly_commit) - Gitlab::Git::Commit.new(commit) + Gitlab::Git::Commit.new(@repository, gitaly_commit) end def between(from, to) @@ -156,7 +155,7 @@ module Gitlab private def commit_diff_request_params(commit, options = {}) - parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + parent_id = commit.parent_ids.first || EMPTY_TREE_ID { repository: @gitaly_repo, @@ -169,8 +168,7 @@ module Gitlab def consume_commits_response(response) response.flat_map do |message| message.commits.map do |gitaly_commit| - commit = GitalyClient::Commit.new(@repository, gitaly_commit) - Gitlab::Git::Commit.new(commit) + Gitlab::Git::Commit.new(@repository, gitaly_commit) end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index b0f7548b7dc..919fb68b8c7 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -16,8 +16,7 @@ module Gitlab response.flat_map do |message| message.branches.map do |branch| - gitaly_commit = GitalyClient::Commit.new(@repository, branch.target) - target_commit = Gitlab::Git::Commit.decorate(gitaly_commit) + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) end end @@ -102,8 +101,7 @@ module Gitlab response.flat_map do |message| message.tags.map do |gitaly_tag| if gitaly_tag.target_commit.present? - commit = GitalyClient::Commit.new(@repository, gitaly_tag.target_commit) - gitaly_commit = Gitlab::Git::Commit.new(commit) + gitaly_commit = Gitlab::Git::Commit.decorate(@repository, gitaly_tag.target_commit) end Gitlab::Git::Tag.new( @@ -141,7 +139,7 @@ module Gitlab committer_email: response.commit_committer.email.dup } - Gitlab::Git::Commit.decorate(hash) + Gitlab::Git::Commit.decorate(@repository, hash) end end end -- cgit v1.2.1 From e363fbf71a7874de2352740b3f33350e5ec4cf54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Tue, 25 Jul 2017 17:26:52 -0400 Subject: Move `deltas` and `diff_from_parents` logic to Gitlab::Git::Commit This helps keep the abstraction layers simpler, and also keep the interface of those methods consistent, in case of implementation changes. --- lib/gitlab/git/commit.rb | 58 ++++++++++++++++++------------ lib/gitlab/git/commit_stats.rb | 2 +- lib/gitlab/gitaly_client/commit_service.rb | 7 ++-- 3 files changed, 39 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index b08d7e8fec3..c3eb3fc44f5 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -187,25 +187,6 @@ module Gitlab Gitlab::Git::Commit.new(repository, commit, ref) end - # Returns a diff object for the changes introduced by +rugged_commit+. - # If +rugged_commit+ doesn't have a parent, then the diff is between - # this commit and an empty repo. See Repository#diff for the keys - # allowed in the +options+ hash. - def diff_from_parent(rugged_commit, options = {}) - options ||= {} - break_rewrites = options[:break_rewrites] - actual_options = Gitlab::Git::Diff.filter_diff_options(options) - - diff = if rugged_commit.parents.empty? - rugged_commit.diff(actual_options.merge(reverse: true)) - else - rugged_commit.parents[0].diff(rugged_commit, actual_options) - end - - diff.find_similar!(break_rewrites: break_rewrites) - diff - end - # Returns the `Rugged` sorting type constant for one or more given # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array # containing more than one of them. `:date` uses a combination of date and @@ -270,19 +251,50 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 def to_diff - diff_from_parent.patch + rugged_diff_from_parent.patch end # Returns a diff object for the changes from this commit's first parent. # If there is no parent, then the diff is between this commit and an - # empty repo. See Repository#diff for keys allowed in the +options+ + # empty repo. See Repository#diff for keys allowed in the +options+ # hash. def diff_from_parent(options = {}) - Commit.diff_from_parent(raw_commit, options) + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| + if is_enabled + @repository.gitaly_commit_client.diff_from_parent(self, options) + else + rugged_diff_from_parent(options) + end + end + end + + def rugged_diff_from_parent(options = {}) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options) + + diff = if raw_commit.parents.empty? + raw_commit.diff(actual_options.merge(reverse: true)) + else + raw_commit.parents[0].diff(raw_commit, actual_options) + end + + diff.find_similar!(break_rewrites: break_rewrites) + diff end def deltas - @deltas ||= diff_from_parent.each_delta.map { |d| Gitlab::Git::Diff.new(d) } + @deltas ||= begin + deltas = Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled| + if is_enabled + @repository.gitaly_commit_client.commit_deltas(self) + else + rugged_diff_from_parent.each_delta + end + end + + deltas.map { |delta| Gitlab::Git::Diff.new(delta) } + end end def has_zero_stats? diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 57c29ad112c..00acb4763e9 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -16,7 +16,7 @@ module Gitlab @deletions = 0 @total = 0 - diff = commit.diff_from_parent + diff = commit.rugged_diff_from_parent diff.each_patch do |p| # TODO: Use the new Rugged convenience methods when they're released diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 2a97a025e58..793de65595f 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -29,15 +29,14 @@ module Gitlab request = Gitaly::CommitDiffRequest.new(request_params) response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) - Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options.merge(from_gitaly: true)) + GitalyClient::DiffStitcher.new(response) end def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) - response.flat_map do |msg| - msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } - end + + response.flat_map { |msg| msg.deltas } end def tree_entry(ref, path, limit = nil) -- cgit v1.2.1 From 3ce6f03f1437633c9328dc30aa5272a49368655b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Tue, 25 Jul 2017 17:33:06 -0400 Subject: Incorporate Gitaly's CommitService.FindCommit RPC --- lib/gitlab/conflict/file_collection.rb | 4 +-- lib/gitlab/git/commit.rb | 52 ++++++++++++++++++++---------- lib/gitlab/gitaly_client.rb | 4 +++ lib/gitlab/gitaly_client/commit_service.rb | 17 ++++++++-- 4 files changed, 55 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 1611eba31da..d671867e7c7 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -77,8 +77,8 @@ EOM def initialize(merge_request, project) @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.raw_commit - @their_commit = merge_request.target_branch_head.raw.raw_commit + @our_commit = merge_request.source_branch_head.raw.rugged_commit + @their_commit = merge_request.target_branch_head.raw.rugged_commit @project = project end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index c3eb3fc44f5..9256663f454 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -14,7 +14,7 @@ module Gitlab attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator - delegate :tree, to: :raw_commit + delegate :tree, to: :rugged_commit def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -50,19 +50,29 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") + # Already a commit? return commit_id if commit_id.is_a?(Gitlab::Git::Commit) + + # A rugged reference? + commit_id = Gitlab::Git::Ref.dereference_object(commit_id) return decorate(repo, commit_id) if commit_id.is_a?(Rugged::Commit) - obj = if commit_id.is_a?(String) - repo.rev_parse_target(commit_id) - else - Gitlab::Git::Ref.dereference_object(commit_id) - end + # Some weird thing? + return nil unless commit_id.is_a?(String) + + commit = repo.gitaly_migrate(:find_commit) do |is_enabled| + if is_enabled + repo.gitaly_commit_client.find_commit(commit_id) + else + obj = repo.rev_parse_target(commit_id) - return nil unless obj.is_a?(Rugged::Commit) + obj.is_a?(Rugged::Commit) ? obj : nil + end + end - decorate(repo, obj) - rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository + decorate(repo, commit) if commit + rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, + Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository nil end @@ -273,11 +283,11 @@ module Gitlab break_rewrites = options[:break_rewrites] actual_options = Gitlab::Git::Diff.filter_diff_options(options) - diff = if raw_commit.parents.empty? - raw_commit.diff(actual_options.merge(reverse: true)) - else - raw_commit.parents[0].diff(raw_commit, actual_options) - end + diff = if rugged_commit.parents.empty? + rugged_commit.diff(actual_options.merge(reverse: true)) + else + rugged_commit.parents[0].diff(rugged_commit, actual_options) + end diff.find_similar!(break_rewrites: break_rewrites) diff @@ -340,7 +350,7 @@ module Gitlab def to_patch(options = {}) begin - raw_commit.to_mbox(options) + rugged_commit.to_mbox(options) rescue Rugged::InvalidError => ex if ex.message =~ /commit \w+ is a merge commit/i 'Patch format is not currently supported for merge commits.' @@ -388,6 +398,14 @@ module Gitlab encode! @committer_email end + def rugged_commit + @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit) + raw_commit + else + @repository.rev_parse_target(id) + end + end + private def init_from_hash(hash) @@ -421,10 +439,10 @@ module Gitlab # subject from the message to make it clearer when there's one # available but not the other. @message = (commit.body.presence || commit.subject).dup - @authored_date = Time.at(commit.author.date.seconds) + @authored_date = Time.at(commit.author.date.seconds).utc @author_name = commit.author.name.dup @author_email = commit.author.email.dup - @committed_date = Time.at(commit.committer.date.seconds) + @committed_date = Time.at(commit.committer.date.seconds).utc @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = commit.parent_ids diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c90ef282fdd..70177cd0fec 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -100,5 +100,9 @@ module Gitlab path = Rails.root.join(SERVER_VERSION_FILE) path.read.chomp end + + def self.encode(s) + s.dup.force_encoding(Encoding::ASCII_8BIT) + end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 793de65595f..fef77f43670 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -43,7 +43,7 @@ module Gitlab request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: ref, - path: path.dup.force_encoding(Encoding::ASCII_8BIT), + path: GitalyClient.encode(path), limit: limit.to_i ) @@ -99,8 +99,8 @@ module Gitlab def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, - revision: revision.force_encoding(Encoding::ASCII_8BIT), - path: path.to_s.force_encoding(Encoding::ASCII_8BIT) + revision: GitalyClient.encode(revision), + path: GitalyClient.encode(path.to_s) ) gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit @@ -151,6 +151,17 @@ module Gitlab response.reduce("") { |memo, msg| memo << msg.data } end + def find_commit(revision) + request = Gitaly::FindCommitRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request) + + response.commit + end + private def commit_diff_request_params(commit, options = {}) -- cgit v1.2.1 From 8e2350ae9514f4b296e1717ca26a6033e9d2aca8 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Thu, 20 Jul 2017 21:02:07 +0800 Subject: Raise encoding confidence threshold to 50 It is recommended that we set this to 50: https://gitlab.com/gitlab-org/gitlab-ce/issues/35098#note_35036746 In this particular issue, the confidence was 42 for Shift JIS, but in fact that's encoded in UTF-8 just with a single bad character. In this case, we shouldn't try to treat it as Shift JIS, but just treat it as UTF-8 and remove invalid bytes. Treating it like Shift JIS would corrupt the whole data. Unfortunately, the diff which would cause this could not be disclosed therefore we can't use it as a test example. --- lib/gitlab/encoding_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 781f9c56a42..8ddc91e341d 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -11,7 +11,7 @@ module Gitlab # obscure encoding with low confidence. # There is a lot more info with this merge request: # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 - ENCODING_CONFIDENCE_THRESHOLD = 40 + ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) return nil unless message.respond_to? :force_encoding -- cgit v1.2.1 From 94d9b545db0dda59020f00cfc17bb7f9d1e4e5a2 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 8 Aug 2017 13:19:23 +0100 Subject: Fix proxy config in Gitlab Pages nginx examples --- lib/support/nginx/gitlab-pages | 5 ++++- lib/support/nginx/gitlab-pages-ssl | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/support/nginx/gitlab-pages b/lib/support/nginx/gitlab-pages index d9746c5c1aa..875c8bcbf3c 100644 --- a/lib/support/nginx/gitlab-pages +++ b/lib/support/nginx/gitlab-pages @@ -18,8 +18,11 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache off; + # The same address as passed to GitLab Pages: `-listen-proxy` - proxy_pass http://localhost:8090/; + proxy_pass http://localhost:8090/; } # Define custom error pages diff --git a/lib/support/nginx/gitlab-pages-ssl b/lib/support/nginx/gitlab-pages-ssl index a1ccf266835..62ed482e2bf 100644 --- a/lib/support/nginx/gitlab-pages-ssl +++ b/lib/support/nginx/gitlab-pages-ssl @@ -67,8 +67,11 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache off; + # The same address as passed to GitLab Pages: `-listen-proxy` - proxy_pass http://localhost:8090/; + proxy_pass http://localhost:8090/; } # Define custom error pages -- cgit v1.2.1 From 5684528081d0ffef9f128cd1b102889f23075dd0 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 8 Aug 2017 15:34:29 +0200 Subject: Unset BUNDLE_GEMFILE when installing Gitaly --- lib/tasks/gitlab/gitaly.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index aaf00bd703a..1f504485e4c 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -21,7 +21,7 @@ namespace :gitlab do create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? - Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT] + [command]) } + Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + [command]) } end end end -- cgit v1.2.1 From e26acdb11bef56360247a73c07be56784c52febe Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 8 Aug 2017 22:20:16 +0800 Subject: Just extend main, rather than include to Kernel Unfortunately rake doesn't have nested context, everything just runs on a main rake object. This is probably due to compatibility issue, but anyway, we could just extend the object. --- lib/tasks/gitlab/check.rake | 12 ------------ lib/tasks/gitlab/helpers.rake | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index dbb3b827b9a..1bd36bbe20a 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -41,8 +41,6 @@ namespace :gitlab do end namespace :gitlab_shell do - include SystemCheck::Helpers - desc "GitLab | Check the configuration of GitLab Shell" task check: :environment do warn_user_is_not_gitlab @@ -249,8 +247,6 @@ namespace :gitlab do end namespace :sidekiq do - include SystemCheck::Helpers - desc "GitLab | Check the configuration of Sidekiq" task check: :environment do warn_user_is_not_gitlab @@ -309,8 +305,6 @@ namespace :gitlab do end namespace :incoming_email do - include SystemCheck::Helpers - desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab @@ -444,8 +438,6 @@ namespace :gitlab do end namespace :ldap do - include SystemCheck::Helpers - task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. @@ -501,8 +493,6 @@ namespace :gitlab do end namespace :repo do - include SystemCheck::Helpers - desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do Gitlab.config.repositories.storages.each do |name, repository_storage| @@ -517,8 +507,6 @@ namespace :gitlab do end namespace :user do - include SystemCheck::Helpers - desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake index dd2d5861481..b0a24790c4a 100644 --- a/lib/tasks/gitlab/helpers.rake +++ b/lib/tasks/gitlab/helpers.rake @@ -4,5 +4,5 @@ require 'tasks/gitlab/task_helpers' StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] namespace :gitlab do - include Gitlab::TaskHelpers + extend SystemCheck::Helpers end -- cgit v1.2.1 From 9770c57fab0315865a33c8b6df269eded0d57b5c Mon Sep 17 00:00:00 2001 From: Brian Neel Date: Thu, 3 Aug 2017 22:20:34 -0400 Subject: Re-enable SqlInjection and CommandInjection --- lib/api/helpers/members_helpers.rb | 4 +++- lib/api/notes.rb | 2 +- lib/ci/ansi2html.rb | 4 ++-- lib/ci/charts.rb | 2 +- lib/gitlab/diff/line.rb | 2 +- lib/gitlab/git/commit.rb | 4 ++-- lib/gitlab/git/diff.rb | 4 ++-- lib/gitlab/gitaly_client/diff.rb | 4 ++-- lib/gitlab/reference_extractor.rb | 2 +- lib/static_model.rb | 2 +- 10 files changed, 16 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index d9cae1501f8..a50ea0b52aa 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -1,8 +1,10 @@ +# rubocop:disable GitlabSecurity/PublicSend + module API module Helpers module MembersHelpers def find_source(source_type, id) - public_send("find_#{source_type}!", id) + public_send("find_#{source_type}!", id) # rubocop:disable GitlabSecurity/PublicSend end def authorize_admin_source!(source_type, source) diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 65ff89edf65..4e4e473994b 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -139,7 +139,7 @@ module API helpers do def find_project_noteable(noteables_str, noteable_id) - public_send("find_project_#{noteables_str.singularize}", noteable_id) + public_send("find_project_#{noteables_str.singularize}", noteable_id) # rubocop:disable GitlabSecurity/PublicSend end def noteable_read_ability_name(noteable) diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 55402101e43..8354fc8d595 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -254,7 +254,7 @@ module Ci def state state = STATE_PARAMS.inject({}) do |h, param| - h[param] = send(param) + h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend h end Base64.urlsafe_encode64(state.to_json) @@ -266,7 +266,7 @@ module Ci return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| - send("#{param}=".to_sym, state[param]) + send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 872e418c788..76a69bf8a83 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -47,7 +47,7 @@ module Ci def collect query = project.pipelines - .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) + .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection totals_count = grouped_count(query) success_count = grouped_count(query.success) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 2d89ccfc354..0603141e441 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -21,7 +21,7 @@ module Gitlab def to_hash hash = {} - serialize_keys.each { |key| hash[key] = send(key) } + serialize_keys.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend hash end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 9256663f454..fd4dfdb09a2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -319,7 +319,7 @@ module Gitlab def to_hash serialize_keys.map.with_object({}) do |key, hash| - hash[key] = send(key) + hash[key] = send(key) # rubocop:disable GitlabSecurity/PublicSend end end @@ -412,7 +412,7 @@ module Gitlab raw_commit = hash.symbolize_keys serialize_keys.each do |key| - send("#{key}=", raw_commit[key]) + send("#{key}=", raw_commit[key]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 9e00abefd02..ce3d65062e8 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -143,7 +143,7 @@ module Gitlab hash = {} SERIALIZE_KEYS.each do |key| - hash[key] = send(key) + hash[key] = send(key) # rubocop:disable GitlabSecurity/PublicSend end hash @@ -221,7 +221,7 @@ module Gitlab raw_diff = hash.symbolize_keys SERIALIZE_KEYS.each do |key| - send(:"#{key}=", raw_diff[key.to_sym]) + send(:"#{key}=", raw_diff[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index d459c9a88fb..54df6304865 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -7,13 +7,13 @@ module Gitlab def initialize(params) params.each do |key, val| - public_send(:"#{key}=", val) + public_send(:"#{key}=", val) # rubocop:disable GitlabSecurity/PublicSend end end def ==(other) FIELDS.all? do |field| - public_send(field) == other.public_send(field) + public_send(field) == other.public_send(field) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index f5b757ace77..bc836dcc08d 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -45,7 +45,7 @@ module Gitlab end def all - REFERABLES.each { |referable| send(referable.to_s.pluralize) } + REFERABLES.each { |referable| send(referable.to_s.pluralize) } # rubocop:disable GitlabSecurity/PublicSend @references.values.flatten end diff --git a/lib/static_model.rb b/lib/static_model.rb index 185921d8fbe..60e2dd82e4e 100644 --- a/lib/static_model.rb +++ b/lib/static_model.rb @@ -18,7 +18,7 @@ module StaticModel # # Pass it along if we respond to it. def [](key) - send(key) if respond_to?(key) + send(key) if respond_to?(key) # rubocop:disable GitlabSecurity/PublicSend end def to_param -- cgit v1.2.1 From 5d963fccab8732dbd5ee54579d3afbcd47131ca6 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 8 Aug 2017 23:14:29 +0800 Subject: We shouldn't include utility methods everywhere --- lib/tasks/gitlab/task_helpers.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index 28b2d86eed2..d85b810ac66 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -5,6 +5,8 @@ module Gitlab TaskAbortedByUserError = Class.new(StandardError) module TaskHelpers + extend self + # Ask if the user wants to continue # # Returns "yes" the user chose to continue -- cgit v1.2.1 From c946ee1282655d332da4ba99c448d6f68cf87cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 9 Aug 2017 11:52:22 +0200 Subject: Enable the Layout/SpaceBeforeBlockBraces cop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/api/entities.rb | 2 +- lib/api/v3/entities.rb | 2 +- lib/backup/manager.rb | 4 ++-- lib/gitlab/gitlab_import/client.rb | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 6ba4005dd0b..3bb1910a441 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -132,7 +132,7 @@ module API expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace, using: 'API::Entities::Namespace' - expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } + expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } expose :avatar_url do |user, options| diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 773f667abe0..4a2e9c9cbb0 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -68,7 +68,7 @@ module API expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace, using: 'API::Entities::Namespace' - expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } + expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :avatar_url do |user, options| user.avatar_url(only_path: false) end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index ca6d6848d41..b9a573d3542 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -198,11 +198,11 @@ module Backup end def archives_to_backup - ARCHIVES_TO_BACKUP.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + ARCHIVES_TO_BACKUP.map { |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup - FOLDERS_TO_BACKUP.reject{ |name| skipped?(name) } + FOLDERS_TO_BACKUP.reject { |name| skipped?(name) } end def disabled_features diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 86fb6c51765..f1007daab5d 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -71,7 +71,7 @@ module Gitlab end def config - Gitlab.config.omniauth.providers.find{|provider| provider.name == "gitlab"} + Gitlab.config.omniauth.providers.find {|provider| provider.name == "gitlab"} end def gitlab_options -- cgit v1.2.1 From ec2d6b495ba6f7de953729d844803d93d7d38c48 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 9 Aug 2017 12:41:20 +0000 Subject: Explain why we use select all for project_url_constrainer.rb --- lib/constraints/project_url_constrainer.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index 4c0aee6c48f..fd7b97d3167 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -6,6 +6,8 @@ class ProjectUrlConstrainer return false unless DynamicPathValidator.valid_project_path?(full_path) + # We intentionally allow SELECT(*) here so result of this query can be used + # as cache for further Project.find_by_full_path calls within request Project.find_by_full_path(full_path, follow_redirects: request.get?).present? end end -- cgit v1.2.1 From b21539cc57148c68aa99ac9ec705d2b1ff2a7b04 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 9 Aug 2017 15:37:05 +0200 Subject: Expose the raw_log method --- lib/gitlab/git/repository.rb | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 371f8797ff2..7000b173075 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -324,6 +324,23 @@ module Gitlab raw_log(options).map { |c| Commit.decorate(self, c) } end + # Used in gitaly-ruby + def raw_log(options) + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + def count_commits(options) gitaly_migrate(:count_commits) do |is_enabled| if is_enabled @@ -733,22 +750,6 @@ module Gitlab sort_branches(branches, sort_by) end - def raw_log(options) - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - def log_using_shell?(options) options[:path].present? || options[:disable_walk] || -- cgit v1.2.1 From 04c328f923afdc4143b875b888235e563b540d90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 8 Aug 2017 13:24:45 +0200 Subject: Fix ee_compat_check when EE branch uses a prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/ee_compat_check.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 72d7d4f84d1..abd401224d8 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -98,10 +98,11 @@ module Gitlab if status.zero? @ee_branch_found = ee_branch_prefix - else - _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}]) + return end + _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}]) + if status.zero? @ee_branch_found = ee_branch_suffix else -- cgit v1.2.1 From 11eda5c255f71f6daf92bd59203e33e121d1d691 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 3 Aug 2017 15:03:06 +0200 Subject: Rename RPC 'Exists' to 'RepositoryExists' --- lib/gitlab/gitaly_client/repository_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 79ce784f2f2..6ad97e62941 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -10,7 +10,7 @@ module Gitlab def exists? request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :exists, request).exists + GitalyClient.call(@storage, :repository_service, :repository_exists, request).exists end def garbage_collect(create_bitmap) -- cgit v1.2.1 From 603b68186a62063802986477c15f5b46694c0100 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 10 Aug 2017 12:28:04 +0100 Subject: Filter sensitive query string parameters from NGINX access logs --- lib/support/nginx/gitlab | 35 ++++++++++++++++++++++++++++++++++- lib/support/nginx/gitlab-ssl | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index f25e66d54c8..54f51d9d633 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -25,6 +25,39 @@ map $http_upgrade $connection_upgrade_gitlab { '' close; } +## NGINX 'combined' log format with filtered query strings +log_format gitlab_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_filtered_http_referer" "$http_user_agent"; + +## Remove private_token from the request URI +# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +map $request_uri $gitlab_temp_request_uri_1 { + default $request_uri; + ~(?i)^(?.*)(?[\?&]private[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest"; +} + +## Remove authenticity_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +map $gitlab_temp_request_uri_1 $gitlab_temp_request_uri_2 { + default $gitlab_temp_request_uri_1; + ~(?i)^(?.*)(?[\?&]authenticity[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest"; +} + +## Remove rss_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&... +map $gitlab_temp_request_uri_2 $gitlab_filtered_request_uri { + default $gitlab_temp_request_uri_2; + ~(?i)^(?.*)(?[\?&]rss[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest"; +} + +## A version of the referer without the query string +map $http_referer $gitlab_filtered_http_referer { + default $http_referer; + ~^(?.*)\? $temp; +} + ## Normal HTTP host server { ## Either remove "default_server" from the listen line below, @@ -46,7 +79,7 @@ server { # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24 ## Individual nginx logs for this GitLab vhost - access_log /var/log/nginx/gitlab_access.log; + access_log /var/log/nginx/gitlab_access.log gitlab_access; error_log /var/log/nginx/gitlab_error.log; location / { diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 2b40da18bab..ed8131ef24f 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -29,6 +29,41 @@ map $http_upgrade $connection_upgrade_gitlab_ssl { '' close; } + +## NGINX 'combined' log format with filtered query strings +log_format gitlab_ssl_access $remote_addr - $remote_user [$time_local] "$request_method $gitlab_ssl_filtered_request_uri $server_protocol" $status $body_bytes_sent "$gitlab_ssl_filtered_http_referer" "$http_user_agent"; + +## Remove private_token from the request URI +# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +map $request_uri $gitlab_ssl_temp_request_uri_1 { + default $request_uri; + ~(?i)^(?.*)(?[\?&]private[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest"; +} + +## Remove authenticity_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +map $gitlab_ssl_temp_request_uri_1 $gitlab_ssl_temp_request_uri_2 { + default $gitlab_ssl_temp_request_uri_1; + ~(?i)^(?.*)(?[\?&]authenticity[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest"; +} + +## Remove rss_token from the request URI +# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&... +# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&... +map $gitlab_ssl_temp_request_uri_2 $gitlab_ssl_filtered_request_uri { + default $gitlab_ssl_temp_request_uri_2; + ~(?i)^(?.*)(?[\?&]rss[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest"; +} + +## A version of the referer without the query string +map $http_referer $gitlab_ssl_filtered_http_referer { + default $http_referer; + ~^(?.*)\? $temp; +} + + ## Redirects all HTTP traffic to the HTTPS host server { ## Either remove "default_server" from the listen line below, @@ -40,7 +75,7 @@ server { server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice return 301 https://$http_host$request_uri; - access_log /var/log/nginx/gitlab_access.log; + access_log /var/log/nginx/gitlab_access.log gitlab_ssl_access; error_log /var/log/nginx/gitlab_error.log; } @@ -93,7 +128,7 @@ server { # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24 ## Individual nginx logs for this GitLab vhost - access_log /var/log/nginx/gitlab_access.log; + access_log /var/log/nginx/gitlab_access.log gitlab_ssl_access; error_log /var/log/nginx/gitlab_error.log; location / { -- cgit v1.2.1 From 7b10885046137633fa615ed5d6ba29d4d0d09cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Fri, 4 Aug 2017 06:16:02 +0200 Subject: Migrate Git::Repository.ls_files to Gitaly --- lib/gitlab/git/repository.rb | 58 ++++++++++++++++++------------ lib/gitlab/gitaly_client/commit_service.rb | 12 +++++++ 2 files changed, 48 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 371f8797ff2..89041997a3a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -603,29 +603,13 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) - actual_ref = ref || root_ref - - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) - cmd += %w(-r) - cmd += %w(--full-tree) - cmd += %w(--full-name) - cmd += %W(-- #{actual_ref}) - - raw_output = IO.popen(cmd, &:read).split("\n").map do |f| - stuff, path = f.split("\t") - _mode, type, _sha = stuff.split(" ") - path if type == "blob" - # Contain only blob type + gitaly_migrate(:ls_files) do |is_enabled| + if is_enabled + gitaly_ls_files(ref) + else + git_ls_files(ref) + end end - - raw_output.compact end # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 @@ -973,6 +957,36 @@ module Gitlab raw_output.to_i end + + def gitaly_ls_files(ref) + gitaly_commit_client.ls_files(ref) + end + + def git_ls_files(ref) + actual_ref = ref || root_ref + + begin + sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) + cmd += %w(-r) + cmd += %w(--full-tree) + cmd += %w(--full-name) + cmd += %W(-- #{actual_ref}) + + raw_output = IO.popen(cmd, &:read).split("\n").map do |f| + stuff, path = f.split("\t") + _mode, type, _sha = stuff.split(" ") + path if type == "blob" + # Contain only blob type + end + + raw_output.compact + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 692d7e02eef..93268d9f33c 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -10,6 +10,18 @@ module Gitlab @repository = repository end + def ls_files(revision) + request = Gitaly::ListFilesRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request) + response.flat_map do |msg| + msg.paths.map { |d| d.dup.force_encoding(Encoding::UTF_8) } + end + end + def is_ancestor(ancestor_id, child_id) request = Gitaly::CommitIsAncestorRequest.new( repository: @gitaly_repo, -- cgit v1.2.1 From 0395c47193b3bbf6b4f060f28c9f632580313a35 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 10 Jul 2017 17:43:57 +0200 Subject: Migrate events into a new format This commit migrates events data in such a way that push events are stored much more efficiently. This is done by creating a shadow table called "events_for_migration", and a table called "push_event_payloads" which is used for storing push data of push events. The background migration in this commit will copy events from the "events" table into the "events_for_migration" table, push events in will also have a row created in "push_event_payloads". This approach allows us to reclaim space in the next release by simply swapping the "events" and "events_for_migration" tables, then dropping the old events (now "events_for_migration") table. The new table structure is also optimised for storage space, and does not include the unused "title" column nor the "data" column (since this data is moved to "push_event_payloads"). == Newly Created Events Newly created events are inserted into both "events" and "events_for_migration", both using the exact same primary key value. The table "push_event_payloads" in turn has a foreign key to the _shadow_ table. This removes the need for recreating and validating the foreign key after swapping the tables. Since the shadow table also has a foreign key to "projects.id" we also don't have to worry about orphaned rows. This approach however does require some additional storage as we're duplicating a portion of the events data for at least 1 release. The exact amount is hard to estimate, but for GitLab.com this is expected to be between 10 and 20 GB at most. The background migration in this commit deliberately does _not_ update the "events" table as doing so would put a lot of pressure on PostgreSQL's auto vacuuming system. == Supporting Both Old And New Events Application code has also been adjusted to support push events using both the old and new data formats. This is done by creating a PushEvent class which extends the regular Event class. Using Rails' Single Table Inheritance system we can ensure the right class is used for the right data, which in this case is based on the value of `events.action`. To support displaying old and new data at the same time the PushEvent class re-defines a few methods of the Event class, falling back to their original implementations for push events in the old format. Once all existing events have been migrated the various push event related methods can be removed from the Event model, and the calls to `super` can be removed from the methods in the PushEvent model. The UI and event atom feed have also been slightly changed to better handle this new setup, fortunately only a few changes were necessary to make this work. == API Changes The API only displays push data of events in the new format. Supporting both formats in the API is a bit more difficult compared to the UI. Since the old push data was not really well documented (apart from one example that used an incorrect "action" nmae) I decided that supporting both was not worth the effort, especially since events will be migrated in a few days _and_ new events are created in the correct format. --- lib/api/entities.rb | 12 +- lib/api/v3/entities.rb | 12 +- .../migrate_events_to_push_event_payloads.rb | 176 +++++++++++++++++++++ lib/gitlab/import_export/import_export.yml | 26 ++- 4 files changed, 216 insertions(+), 10 deletions(-) create mode 100644 lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 3bb1910a441..18cd604a216 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -497,14 +497,24 @@ module API expose :author, using: Entities::UserBasic end + class PushEventPayload < Grape::Entity + expose :commit_count, :action, :ref_type, :commit_from, :commit_to + expose :ref, :commit_title + end + class Event < Grape::Entity expose :title, :project_id, :action_name expose :target_id, :target_iid, :target_type, :author_id - expose :data, :target_title + expose :target_title expose :created_at expose :note, using: Entities::Note, if: ->(event, options) { event.note? } expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author } + expose :push_event_payload, + as: :push_data, + using: PushEventPayload, + if: -> (event, _) { event.push? } + expose :author_username do |event, options| event.author&.username end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 4a2e9c9cbb0..a9a35f2a4bd 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -25,14 +25,24 @@ module API expose(:downvote?) { |note| false } end + class PushEventPayload < Grape::Entity + expose :commit_count, :action, :ref_type, :commit_from, :commit_to + expose :ref, :commit_title + end + class Event < Grape::Entity expose :title, :project_id, :action_name expose :target_id, :target_type, :author_id - expose :data, :target_title + expose :target_title expose :created_at expose :note, using: Entities::Note, if: ->(event, options) { event.note? } expose :author, using: ::API::Entities::UserBasic, if: ->(event, options) { event.author } + expose :push_event_payload, + as: :push_data, + using: PushEventPayload, + if: -> (event, _) { event.push? } + expose :author_username do |event, options| event.author&.username end diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb new file mode 100644 index 00000000000..432f7c3e706 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -0,0 +1,176 @@ +module Gitlab + module BackgroundMigration + # Class that migrates events for the new push event payloads setup. All + # events are copied to a shadow table, and push events will also have a row + # created in the push_event_payloads table. + class MigrateEventsToPushEventPayloads + class Event < ActiveRecord::Base + self.table_name = 'events' + + serialize :data + + BLANK_REF = ('0' * 40).freeze + TAG_REF_PREFIX = 'refs/tags/'.freeze + MAX_INDEX = 69 + PUSHED = 5 + + def push_event? + action == PUSHED && data.present? + end + + def commit_title + commit = commits.last + + return nil unless commit && commit[:message] + + index = commit[:message].index("\n") + message = index ? commit[:message][0..index] : commit[:message] + + message.strip.truncate(70) + end + + def commit_from_sha + if create? + nil + else + data[:before] + end + end + + def commit_to_sha + if remove? + nil + else + data[:after] + end + end + + def data + super || {} + end + + def commits + data[:commits] || [] + end + + def commit_count + data[:total_commits_count] || 0 + end + + def ref + data[:ref] + end + + def trimmed_ref_name + if ref_type == :tag + ref[10..-1] + else + ref[11..-1] + end + end + + def create? + data[:before] == BLANK_REF + end + + def remove? + data[:after] == BLANK_REF + end + + def push_action + if create? + :created + elsif remove? + :removed + else + :pushed + end + end + + def ref_type + if ref.start_with?(TAG_REF_PREFIX) + :tag + else + :branch + end + end + end + + class EventForMigration < ActiveRecord::Base + self.table_name = 'events_for_migration' + end + + class PushEventPayload < ActiveRecord::Base + self.table_name = 'push_event_payloads' + + enum action: { + created: 0, + removed: 1, + pushed: 2 + } + + enum ref_type: { + branch: 0, + tag: 1 + } + end + + # start_id - The start ID of the range of events to process + # end_id - The end ID of the range to process. + def perform(start_id, end_id) + return unless migrate? + + find_events(start_id, end_id).each { |event| process_event(event) } + end + + def process_event(event) + replicate_event(event) + create_push_event_payload(event) if event.push_event? + end + + def replicate_event(event) + new_attributes = event.attributes + .with_indifferent_access.except(:title, :data) + + EventForMigration.create!(new_attributes) + rescue ActiveRecord::InvalidForeignKey + # A foreign key error means the associated event was removed. In this + # case we'll just skip migrating the event. + end + + def create_push_event_payload(event) + commit_from = pack(event.commit_from_sha) + commit_to = pack(event.commit_to_sha) + + PushEventPayload.create!( + event_id: event.id, + commit_count: event.commit_count, + ref_type: event.ref_type, + action: event.push_action, + commit_from: commit_from, + commit_to: commit_to, + ref: event.trimmed_ref_name, + commit_title: event.commit_title + ) + rescue ActiveRecord::InvalidForeignKey + # A foreign key error means the associated event was removed. In this + # case we'll just skip migrating the event. + end + + def find_events(start_id, end_id) + Event + .where('NOT EXISTS (SELECT true FROM events_for_migration WHERE events_for_migration.id = events.id)') + .where(id: start_id..end_id) + end + + def migrate? + Event.table_exists? && PushEventPayload.table_exists? && + EventForMigration.table_exists? + end + + def pack(value) + value ? [value].pack('H*') : nil + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index c5c05bfe2fb..9d9ebcb389a 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -3,18 +3,22 @@ project_tree: - labels: :priorities - milestones: - - :events + - events: + - :push_event_payload - issues: - - :events + - events: + - :push_event_payload - :timelogs - notes: - :author - - :events + - events: + - :push_event_payload - label_links: - label: :priorities - milestone: - - :events + - events: + - :push_event_payload - snippets: - :award_emoji - notes: @@ -25,21 +29,25 @@ project_tree: - merge_requests: - notes: - :author - - :events + - events: + - :push_event_payload - merge_request_diff: - :merge_request_diff_commits - :merge_request_diff_files - - :events + - events: + - :push_event_payload - :timelogs - label_links: - label: :priorities - milestone: - - :events + - events: + - :push_event_payload - pipelines: - notes: - :author - - :events + - events: + - :push_event_payload - :stages - :statuses - :triggers @@ -107,6 +115,8 @@ excluded_attributes: statuses: - :trace - :token + push_event_payload: + - :event_id methods: labels: -- cgit v1.2.1 From aac1de46c9be659b74da12f704412f38292974db Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 27 Jul 2017 19:42:15 +0200 Subject: Use a specialized class for querying events This changes various controllers to use the new EventCollection class for retrieving events. This class uses a JOIN LATERAL query on PostgreSQL to retrieve queries in a more efficient way, while falling back to a simpler / less efficient query for MySQL. The EventCollection class also includes a limit on the number of events to display to prevent malicious users from cycling through all events, as doing so could put a lot of pressure on the database. JOIN LATERAL is only supported on PostgreSQL starting with version 9.3.0 and as such this optimisation is only used when using PostgreSQL 9.3 or newer. --- lib/gitlab/database.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d7dab584a44..e001d25e7b7 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -25,6 +25,10 @@ module Gitlab database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end + def self.join_lateral_supported? + postgresql? && version.to_f >= 9.3 + end + def self.nulls_last_order(field, direction = 'ASC') order = "#{field} #{direction}" -- cgit v1.2.1 From 334915d50884e54ed8034b4b8820f285b14837c5 Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Tue, 8 Aug 2017 21:18:02 +0000 Subject: Merge branch 'import-symlinks-9-3' into 'security-9-3' Fix file disclosure via hidden symlinks using the project import (9.3) See merge request !2164 --- lib/gitlab/import_export/file_importer.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index ffd17118c91..989342389bc 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -47,12 +47,16 @@ module Gitlab end def remove_symlinks! - Dir["#{@shared.export_path}/**/*"].each do |path| + extracted_files.each do |path| FileUtils.rm(path) if File.lstat(path).symlink? end true end + + def extracted_files + Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ /.*\/\.{1,2}$/ } + end end end end -- cgit v1.2.1 From b29692168184cef044c6a1b244f791c56c10fb1c Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Tue, 8 Aug 2017 17:36:24 +0000 Subject: Merge branch 'rs-alphanumeric-ssh-params' into 'security-9-4' Ensure user and hostnames begin with an alnum character in UrlBlocker See merge request !2138 --- lib/gitlab/url_blocker.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 7e14a566696..fee1a127fd7 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -19,6 +19,8 @@ module Gitlab return false if internal?(uri) return true if blocked_port?(uri.port) + return true if blocked_user_or_hostname?(uri.user) + return true if blocked_user_or_hostname?(uri.hostname) server_ips = Resolv.getaddresses(uri.hostname) return true if (blocked_ips & server_ips).any? @@ -37,6 +39,12 @@ module Gitlab port < 1024 && !VALID_PORTS.include?(port) end + def blocked_user_or_hostname?(value) + return false if value.blank? + + value !~ /\A\p{Alnum}/ + end + def internal?(uri) internal_web?(uri) || internal_shell?(uri) end -- cgit v1.2.1 From 6cd9888f6fc8bb1e0b6ff11ace8aacb19aedb268 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 9 Aug 2017 16:43:10 +0200 Subject: store gpg return directory locally --- lib/gitlab/gpg.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index e1d1724295a..653c56d925b 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -44,19 +44,23 @@ module Gitlab def using_tmp_keychain Dir.mktmpdir do |dir| - @original_dirs ||= [GPGME::Engine.dirinfo('homedir')] - @original_dirs.push(dir) + previous_dir = current_home_dir GPGME::Engine.home_dir = dir return_value = yield - @original_dirs.pop - - GPGME::Engine.home_dir = @original_dirs[-1] + GPGME::Engine.home_dir = previous_dir return_value end end + + # 1. Returns the custom home directory if one has been set by calling + # `GPGME::Engine.home_dir=` + # 2. Returns the default home directory otherwise + def current_home_dir + GPGME::Engine.info.first.home_dir || GPGME::Engine.dirinfo('homedir') + end end end -- cgit v1.2.1 From d2315054e6e8398fade9795f72830550b03fda0f Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 11 Aug 2017 11:58:21 +0100 Subject: Fix merge request diff deserialisation when too_large was absent Not all diffs in st_diffs had the too_large attribute set at all, but the column is non-nullable. Explicitly cast to boolean to avoid that problem. --- .../background_migration/deserialize_merge_request_diffs_and_commits.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 0fbc6b70989..310a69a4bd4 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -85,6 +85,8 @@ module Gitlab diff_hash.tap do |hash| diff_text = hash[:diff] + hash[:too_large] = !!hash[:too_large] + if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? hash[:binary] = true hash[:diff] = [diff_text].pack('m0') -- cgit v1.2.1 From 4f0fa13eb85994b747c1eb253e346b76b98b5c5b Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Fri, 11 Aug 2017 20:50:35 +0800 Subject: Show error message for API 500 error in tests, and document have_gitlab_http_status --- lib/api/helpers.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 99b8b62691f..3582ed81b0f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -257,7 +257,15 @@ module API message << " " << trace.join("\n ") API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + + response_message = + if Rails.env.test? + message + else + '500 Internal Server Error' + end + + rack_response({ 'message' => response_message }.to_json, 500) end # project helpers -- cgit v1.2.1 From 8f8fd342313b0cd459d2fedb5b461b0cc063f248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 19 Jul 2017 19:51:59 +0200 Subject: Use a new RspecFlakyListener to detect flaky specs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/rspec_flaky/example.rb | 46 ++++++++++++++++++++++++ lib/rspec_flaky/flaky_example.rb | 39 +++++++++++++++++++++ lib/rspec_flaky/listener.rb | 75 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 lib/rspec_flaky/example.rb create mode 100644 lib/rspec_flaky/flaky_example.rb create mode 100644 lib/rspec_flaky/listener.rb (limited to 'lib') diff --git a/lib/rspec_flaky/example.rb b/lib/rspec_flaky/example.rb new file mode 100644 index 00000000000..b6e790cbbab --- /dev/null +++ b/lib/rspec_flaky/example.rb @@ -0,0 +1,46 @@ +module RspecFlaky + # This is a wrapper class for RSpec::Core::Example + class Example + delegate :status, :exception, to: :execution_result + + def initialize(rspec_example) + @rspec_example = rspec_example.try(:example) || rspec_example + end + + def uid + @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") + end + + def example_id + rspec_example.id + end + + def file + metadata[:file_path] + end + + def line + metadata[:line_number] + end + + def description + metadata[:full_description] + end + + def attempts + rspec_example.try(:attempts) || 1 + end + + private + + attr_reader :rspec_example + + def metadata + rspec_example.metadata + end + + def execution_result + rspec_example.execution_result + end + end +end diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb new file mode 100644 index 00000000000..f81fb90e870 --- /dev/null +++ b/lib/rspec_flaky/flaky_example.rb @@ -0,0 +1,39 @@ +module RspecFlaky + # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file + class FlakyExample < OpenStruct + def initialize(example) + if example.respond_to?(:example_id) + super( + example_id: example.example_id, + file: example.file, + line: example.line, + description: example.description, + last_attempts_count: example.attempts, + flaky_reports: 1) + else + super + end + end + + def first_flaky_at + self[:first_flaky_at] || Time.now + end + + def last_flaky_at + Time.now + end + + def last_flaky_job + return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] + + "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + end + + def to_h + super.merge( + first_flaky_at: first_flaky_at, + last_flaky_at: last_flaky_at, + last_flaky_job: last_flaky_job) + end + end +end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb new file mode 100644 index 00000000000..ec2fbd9e36c --- /dev/null +++ b/lib/rspec_flaky/listener.rb @@ -0,0 +1,75 @@ +require 'json' + +module RspecFlaky + class Listener + attr_reader :all_flaky_examples, :new_flaky_examples + + def initialize + @new_flaky_examples = {} + @all_flaky_examples = init_all_flaky_examples + end + + def example_passed(notification) + current_example = RspecFlaky::Example.new(notification.example) + + return unless current_example.attempts > 1 + + flaky_example_hash = all_flaky_examples[current_example.uid] + + all_flaky_examples[current_example.uid] = + if flaky_example_hash + FlakyExample.new(flaky_example_hash).tap do |ex| + ex.last_attempts_count = current_example.attempts + ex.flaky_reports += 1 + end + else + FlakyExample.new(current_example).tap do |ex| + new_flaky_examples[current_example.uid] = ex + end + end + end + + def dump_summary(_) + write_report_file(all_flaky_examples, all_flaky_examples_report_path) + + if new_flaky_examples.any? + Rails.logger.warn "\nNew flaky examples detected:\n" + Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples)) + + write_report_file(new_flaky_examples, new_flaky_examples_report_path) + end + end + + def to_report(examples) + Hash[examples.map { |k, ex| [k, ex.to_h] }] + end + + private + + def init_all_flaky_examples + return {} unless File.exist?(all_flaky_examples_report_path) + + all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path)) + + Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }] + end + + def write_report_file(examples, file_path) + return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + + report_path_dir = File.dirname(file_path) + FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) + File.write(file_path, JSON.pretty_generate(to_report(examples))) + end + + def all_flaky_examples_report_path + @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] || + Rails.root.join("rspec_flaky/all-report.json") + end + + def new_flaky_examples_report_path + @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || + Rails.root.join("rspec_flaky/new-report.json") + end + end +end -- cgit v1.2.1 From 09a348eb139178be534d181273a360a3125df9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 11 Aug 2017 14:08:20 +0200 Subject: Include the `is_admin` field in the `GET /users/:id` API when current user is an admin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/api/users.rb | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index a590f2692a2..e2019d6d512 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -79,22 +79,17 @@ module API end desc 'Get a single user' do - success Entities::UserBasic + success Entities::User end params do requires :id, type: Integer, desc: 'The ID of the user' end get ":id" do user = User.find_by(id: params[:id]) - not_found!('User') unless user + not_found!('User') unless user && can?(current_user, :read_user, user) - if current_user && current_user.admin? - present user, with: Entities::UserPublic - elsif can?(current_user, :read_user, user) - present user, with: Entities::User - else - render_api_error!("User not found.", 404) - end + opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : {} + present user, opts end desc 'Create a user. Available only for admins.' do -- cgit v1.2.1 From fab0c1eb80b3eda00024a4e1fb961ba5b8bcc7bb Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 10 Aug 2017 18:24:44 +0200 Subject: Use existing BUNDLE_PATH for gitaly in local tests --- lib/tasks/gitlab/gitaly.rake | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 1f504485e4c..e337c67a0f5 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -15,13 +15,17 @@ namespace :gitlab do checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) - command = status.zero? ? 'gmake' : 'make' + command = status.zero? ? ['gmake'] : ['make'] + + if Rails.env.test? + command += %W[BUNDLE_PATH=#{Bundler.bundle_path}] + end Dir.chdir(args.dir) do create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? - Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + [command]) } + Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + command) } end end end -- cgit v1.2.1 From d0622b79d8d011c80f63e71c96e69754a5b0ec16 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 10 Aug 2017 19:09:14 -0400 Subject: Better categorize test coverage results Also marks a few things as uncovered, and removes an unused class. --- lib/file_streamer.rb | 16 ---------------- lib/gitlab/middleware/webpack_proxy.rb | 2 ++ lib/gitlab/o_auth/session.rb | 2 ++ lib/gitlab/seeder.rb | 2 ++ 4 files changed, 6 insertions(+), 16 deletions(-) delete mode 100644 lib/file_streamer.rb (limited to 'lib') diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb deleted file mode 100644 index 4e3c6d3c773..00000000000 --- a/lib/file_streamer.rb +++ /dev/null @@ -1,16 +0,0 @@ -class FileStreamer #:nodoc: - attr_reader :to_path - - def initialize(path) - @to_path = path - end - - # Stream the file's contents if Rack::Sendfile isn't present. - def each - File.open(to_path, 'rb') do |file| - while chunk = file.read(16384) - yield chunk - end - end - end -end diff --git a/lib/gitlab/middleware/webpack_proxy.rb b/lib/gitlab/middleware/webpack_proxy.rb index 6105d165810..6aecf63231f 100644 --- a/lib/gitlab/middleware/webpack_proxy.rb +++ b/lib/gitlab/middleware/webpack_proxy.rb @@ -1,6 +1,7 @@ # This Rack middleware is intended to proxy the webpack assets directory to the # webpack-dev-server. It is only intended for use in development. +# :nocov: module Gitlab module Middleware class WebpackProxy < Rack::Proxy @@ -22,3 +23,4 @@ module Gitlab end end end +# :nocov: diff --git a/lib/gitlab/o_auth/session.rb b/lib/gitlab/o_auth/session.rb index f33bfd0bd0e..30739f2a2c5 100644 --- a/lib/gitlab/o_auth/session.rb +++ b/lib/gitlab/o_auth/session.rb @@ -1,3 +1,4 @@ +# :nocov: module Gitlab module OAuth module Session @@ -15,3 +16,4 @@ module Gitlab end end end +# :nocov: diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 823f697f51c..f9ab9bd466f 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -1,3 +1,4 @@ +# :nocov: module DeliverNever def deliver_later self @@ -21,3 +22,4 @@ module Gitlab end end end +# :nocov: -- cgit v1.2.1 From 2ea8442ff3398a788b1005a825c1d13f61f91c2d Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 7 Aug 2017 20:01:45 +0200 Subject: Move the personal snippet uploads from `system` to `-/system` Update the markdown unconditionally since the move might have been done before, but the markdown not updated. --- .../move_personal_snippet_files.rb | 79 ++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 lib/gitlab/background_migration/move_personal_snippet_files.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb new file mode 100644 index 00000000000..07cec96bcc3 --- /dev/null +++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb @@ -0,0 +1,79 @@ +module Gitlab + module BackgroundMigration + class MovePersonalSnippetFiles + delegate :select_all, :execute, :quote_string, to: :connection + + def perform(relative_source, relative_destination) + @source_relative_location = relative_source + @destination_relative_location = relative_destination + + move_personal_snippet_files + end + + def move_personal_snippet_files + query = "SELECT uploads.path, uploads.model_id FROM uploads "\ + "INNER JOIN snippets ON snippets.id = uploads.model_id WHERE uploader = 'PersonalFileUploader'" + select_all(query).each do |upload| + secret = upload['path'].split('/')[0] + file_name = upload['path'].split('/')[1] + + move_file(upload['model_id'], secret, file_name) + update_markdown(upload['model_id'], secret, file_name) + end + end + + def move_file(snippet_id, secret, file_name) + source_dir = File.join(base_directory, @source_relative_location, snippet_id.to_s, secret) + destination_dir = File.join(base_directory, @destination_relative_location, snippet_id.to_s, secret) + + source_file_path = File.join(source_dir, file_name) + destination_file_path = File.join(destination_dir, file_name) + + unless File.exist?(source_file_path) + say "Source file `#{source_file_path}` doesn't exist. Skipping." + return + end + + say "Moving file #{source_file_path} -> #{destination_file_path}" + + FileUtils.mkdir_p(destination_dir) + FileUtils.move(source_file_path, destination_file_path) + end + + def update_markdown(snippet_id, secret, file_name) + source_markdown_path = File.join(@source_relative_location, snippet_id.to_s, secret, file_name) + destination_markdown_path = File.join(@destination_relative_location, snippet_id.to_s, secret, file_name) + + source_markdown = "](#{source_markdown_path})" + destination_markdown = "](#{destination_markdown_path})" + quoted_source = quote_string(source_markdown) + quoted_destination = quote_string(destination_markdown) + + execute("UPDATE snippets "\ + "SET description = replace(snippets.description, '#{quoted_source}', '#{quoted_destination}'), description_html = NULL "\ + "WHERE id = #{snippet_id}") + + query = "SELECT id, note FROM notes WHERE noteable_id = #{snippet_id} "\ + "AND noteable_type = 'Snippet' AND note IS NOT NULL" + select_all(query).each do |note| + text = note['note'].gsub(source_markdown, destination_markdown) + quoted_text = quote_string(text) + + execute("UPDATE notes SET note = '#{quoted_text}', note_html = NULL WHERE id = #{note['id']}") + end + end + + def base_directory + File.join(Rails.root, 'public') + end + + def connection + ActiveRecord::Base.connection + end + + def say(message) + Rails.logger.debug(message) + end + end + end +end -- cgit v1.2.1 From 649d042dbc9e2bfda96bb98b0eabd4b00ea2daff Mon Sep 17 00:00:00 2001 From: Robin Bobbitt Date: Mon, 31 Jul 2017 17:34:47 -0400 Subject: Add option to disable project export on instance --- lib/api/settings.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d55a61fa638..667ba468ce6 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -29,6 +29,7 @@ module API desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :project_export_enabled, type: Boolean, desc: 'Enable project export' optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' -- cgit v1.2.1 From daa0137b41eeec4008e8292971601fc8594b5329 Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Wed, 9 Aug 2017 08:36:24 +0200 Subject: Migrate force push check to Gitaly --- lib/gitlab/checks/force_push.rb | 19 +++++++++++++------ lib/gitlab/gitaly_client/util.rb | 4 +++- 2 files changed, 16 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 1e73f89158d..714464fd5e7 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -5,12 +5,19 @@ module Gitlab return false if project.empty_repo? # Created or deleted branch - if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) - false - else - Gitlab::Git::RevList.new( - path_to_repo: project.repository.path_to_repo, - oldrev: oldrev, newrev: newrev).missed_ref.present? + return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) + + GitalyClient.migrate(:force_push) do |is_enabled| + if is_enabled + !project + .repository + .gitaly_commit_client + .is_ancestor(oldrev, newrev) + else + Gitlab::Git::RevList.new( + path_to_repo: project.repository.path_to_repo, + oldrev: oldrev, newrev: newrev).missed_ref.present? + end end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index f5a4c5493ef..8fc937496af 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -5,7 +5,9 @@ module Gitlab def repository(repository_storage, relative_path) Gitaly::Repository.new( storage_name: repository_storage, - relative_path: relative_path + relative_path: relative_path, + git_object_directory: Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].to_s, + git_alternate_object_directories: Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']) ) end end -- cgit v1.2.1 From d9b6fd4fba1fa996ba6c71358ad933ba2328ba18 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 11 Aug 2017 18:23:47 +0200 Subject: Handle missing .gitmodules when getting submodule urls --- lib/gitlab/git/repository.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7000b173075..081423eb0db 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -827,6 +827,8 @@ module Gitlab return unless commit_object && commit_object.type == :COMMIT gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) + return unless gitmodules + found_module = GitmodulesParser.new(gitmodules.data).parse[path] found_module && found_module['url'] -- cgit v1.2.1 From edcc488b75d8c6fcad3994bcda30a82756496969 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 10 Aug 2017 20:33:24 +0200 Subject: use mutex for keychain interaction setting of the gpg home directory is not thread safe, as the directoy gets stored on the class. if multiple threads change the directory at the same time, one of the threads will be working in the wrong directory. --- lib/gitlab/gpg.rb | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 653c56d925b..78ebd8866a1 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -2,6 +2,8 @@ module Gitlab module Gpg extend self + MUTEX = Mutex.new + module CurrentKeyChain extend self @@ -42,7 +44,30 @@ module Gitlab end end - def using_tmp_keychain + # Allows thread safe switching of temporary keychain files + # + # 1. The current thread may use nesting of temporary keychain + # 2. Another thread needs to wait for the lock to be released + def using_tmp_keychain(&block) + if MUTEX.locked? && MUTEX.owned? + optimistic_using_tmp_keychain(&block) + else + MUTEX.synchronize do + optimistic_using_tmp_keychain(&block) + end + end + end + + # 1. Returns the custom home directory if one has been set by calling + # `GPGME::Engine.home_dir=` + # 2. Returns the default home directory otherwise + def current_home_dir + GPGME::Engine.info.first.home_dir || GPGME::Engine.dirinfo('homedir') + end + + private + + def optimistic_using_tmp_keychain Dir.mktmpdir do |dir| previous_dir = current_home_dir @@ -55,12 +80,5 @@ module Gitlab return_value end end - - # 1. Returns the custom home directory if one has been set by calling - # `GPGME::Engine.home_dir=` - # 2. Returns the default home directory otherwise - def current_home_dir - GPGME::Engine.info.first.home_dir || GPGME::Engine.dirinfo('homedir') - end end end -- cgit v1.2.1 From a175966677edc385156eb9dab79d129ece0bb87f Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Mon, 14 Aug 2017 12:38:08 +0200 Subject: reset original directory in ensure --- lib/gitlab/gpg.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 78ebd8866a1..45e9f9d65ae 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -68,17 +68,13 @@ module Gitlab private def optimistic_using_tmp_keychain + previous_dir = current_home_dir Dir.mktmpdir do |dir| - previous_dir = current_home_dir - GPGME::Engine.home_dir = dir - - return_value = yield - - GPGME::Engine.home_dir = previous_dir - - return_value + yield end + ensure + GPGME::Engine.home_dir = previous_dir end end end -- cgit v1.2.1 From aef9f1eb9405e9bab92b15f5c99bf06eaf28a5d6 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 14 Aug 2017 15:22:09 +0200 Subject: Cache the number of forks of a project The number of forks of a project doesn't change very frequently and running a COUNT(*) every time this information is requested can be quite expensive. We also end up running such a COUNT(*) query at least twice on the homepage of a project. By caching this data and refreshing it when necessary we can reduce project homepage loading times by around 60 milliseconds (based on the timings of https://gitlab.com/gitlab-org/gitlab-ce). --- lib/api/projects.rb | 2 ++ lib/api/v3/projects.rb | 2 ++ 2 files changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 89dda88d3f5..15c3832b032 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -351,6 +351,8 @@ module API if user_project.forked_from_project.nil? user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + + ::Projects::ForksCountService.new(forked_from_project).refresh_cache else render_api_error!("Project already forked", 409) end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index eb090453b48..449876c10d9 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -388,6 +388,8 @@ module API if user_project.forked_from_project.nil? user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + + ::Projects::ForksCountService.new(forked_from_project).refresh_cache else render_api_error!("Project already forked", 409) end -- cgit v1.2.1 From 260c8da060a6039cbd47cfe31c8ec6d6f9b43de0 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 10 Aug 2017 12:39:26 -0400 Subject: Whitelist or fix additional `Gitlab/PublicSend` cop violations An upcoming update to rubocop-gitlab-security added additional violations. --- lib/api/api_guard.rb | 2 +- lib/api/entities.rb | 5 +++-- lib/api/runners.rb | 2 +- lib/api/v3/notes.rb | 6 +++--- lib/banzai/filter/external_issue_reference_filter.rb | 4 ++-- lib/banzai/object_renderer.rb | 2 +- lib/banzai/pipeline/base_pipeline.rb | 2 +- lib/banzai/renderer.rb | 4 ++-- lib/bitbucket/collection.rb | 2 +- lib/ci/ansi2html.rb | 2 +- lib/declarative_policy/base.rb | 2 +- lib/declarative_policy/dsl.rb | 2 +- lib/file_size_validator.rb | 4 ++-- lib/gitlab/auth.rb | 4 ++-- lib/gitlab/cache/request_cache.rb | 2 +- lib/gitlab/diff/line_mapper.rb | 6 +++--- lib/gitlab/git/blob.rb | 2 +- lib/gitlab/git/tree.rb | 2 +- lib/gitlab/gitaly_client.rb | 2 +- lib/gitlab/github_import/base_formatter.rb | 4 +++- lib/gitlab/github_import/client.rb | 2 +- lib/gitlab/github_import/importer.rb | 2 +- lib/gitlab/lazy.rb | 2 +- lib/gitlab/ldap/person.rb | 4 ++-- lib/gitlab/markdown/pipeline.rb | 2 +- lib/uploaded_file.rb | 2 +- 26 files changed, 39 insertions(+), 36 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 0d2d71e336a..c4c0fdda665 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -122,7 +122,7 @@ module API error_classes = [MissingTokenError, TokenNotFoundError, ExpiredError, RevokedError, InsufficientScopeError] - base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler + base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend end def oauth2_bearer_token_error_handler diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 18cd604a216..716e3f11744 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -541,8 +541,9 @@ module API target_url = "namespace_project_#{target_type}_url" target_anchor = "note_#{todo.note_id}" if todo.note_id? - Gitlab::Routing.url_helpers.public_send(target_url, - todo.project.namespace, todo.project, todo.target, anchor: target_anchor) + Gitlab::Routing + .url_helpers + .public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend end expose :body diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 5bf5a18e42f..31f940fe96b 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -153,7 +153,7 @@ module API render_api_error!('Scope contains invalid value', 400) end - runners.send(scope) + runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend end def get_runner(id) diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb index 23fe95e42e4..d49772b92f2 100644 --- a/lib/api/v3/notes.rb +++ b/lib/api/v3/notes.rb @@ -22,7 +22,7 @@ module API use :pagination end get ":id/#{noteables_str}/:noteable_id/notes" do - noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend if can?(current_user, noteable_read_ability_name(noteable), noteable) # We exclude notes that are cross-references and that cannot be viewed @@ -50,7 +50,7 @@ module API requires :noteable_id, type: Integer, desc: 'The ID of the noteable' end get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend note = noteable.notes.find(params[:note_id]) can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) @@ -76,7 +76,7 @@ module API noteable_id: params[:noteable_id] } - noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + noteable = user_project.public_send(noteables_str.to_sym).find(params[:noteable_id]) # rubocop:disable GitlabSecurity/PublicSend if can?(current_user, noteable_read_ability_name(noteable), noteable) if params[:created_at] && (current_user.admin? || user_project.owner == current_user) diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 53a229256a5..ed01a72ff9f 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -95,10 +95,10 @@ module Banzai private def external_issues_cached(attribute) - return project.public_send(attribute) unless RequestStore.active? + return project.public_send(attribute) unless RequestStore.active? # rubocop:disable GitlabSecurity/PublicSend cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } - cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? + cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend cached_attributes[project.id][attribute] end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 002a3341ccd..2196a92474c 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -37,7 +37,7 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] - object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) + object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) # rubocop:disable GitlabSecurity/PublicSend object.user_visible_reference_count = redacted_data[:visible_reference_count] end end diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index 321fd5bbe14..3ae3bed570d 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -18,7 +18,7 @@ module Banzai define_method(meth) do |text, context| context = transform_context(context) - html_pipeline.send(meth, text, context) + html_pipeline.__send__(meth, text, context) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ad08c0905e2..95d82d17658 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -43,7 +43,7 @@ module Banzai # Same as +render_field+, but without consulting or updating the cache field def self.cacheless_render_field(object, field, options = {}) - text = object.__send__(field) + text = object.__send__(field) # rubocop:disable GitlabSecurity/PublicSend context = object.banzai_render_context(field).merge(options) cacheless_render(text, context) @@ -156,7 +156,7 @@ module Banzai # method. def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key - Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) + Rails.cache.__send__(:expanded_key, full_cache_key(cache_key, pipeline_name)) # rubocop:disable GitlabSecurity/PublicSend end # GitLab EE needs to disable updates on GET requests in Geo diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb index 3a9379ff680..a78495dbf5e 100644 --- a/lib/bitbucket/collection.rb +++ b/lib/bitbucket/collection.rb @@ -13,7 +13,7 @@ module Bitbucket def method_missing(method, *args) return super unless self.respond_to?(method) - self.send(method, *args) do |item| + self.__send__(method, *args) do |item| # rubocop:disable GitlabSecurity/PublicSend block_given? ? yield(item) : item end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 8354fc8d595..b9e9f9f7f4a 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -208,7 +208,7 @@ module Ci return unless command = stack.shift() if self.respond_to?("on_#{command}", true) - self.send("on_#{command}", stack) + self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend end evaluate_command_stack(stack) diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index df94cafb6a1..e544aefa63a 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -109,7 +109,7 @@ module DeclarativePolicy name = name.to_sym if delegation_block.nil? - delegation_block = proc { @subject.__send__(name) } + delegation_block = proc { @subject.__send__(name) } # rubocop:disable GitlabSecurity/PublicSend end own_delegations[name] = delegation_block diff --git a/lib/declarative_policy/dsl.rb b/lib/declarative_policy/dsl.rb index b26807a7622..6ba1e7a3c5c 100644 --- a/lib/declarative_policy/dsl.rb +++ b/lib/declarative_policy/dsl.rb @@ -93,7 +93,7 @@ module DeclarativePolicy def method_missing(m, *a, &b) return super unless @context_class.respond_to?(m) - @context_class.__send__(m, *a, &b) + @context_class.__send__(m, *a, &b) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(m) diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index eb19ab45ac3..de391de9059 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -44,13 +44,13 @@ class FileSizeValidator < ActiveModel::EachValidator when Integer check_value when Symbol - record.send(check_value) + record.public_send(check_value) # rubocop:disable GitlabSecurity/PublicSend end value ||= [] if key == :maximum value_size = value.size - next if value_size.send(validity_check, check_value) + next if value_size.public_send(validity_check, check_value) # rubocop:disable GitlabSecurity/PublicSend errors_options = options.except(*RESERVED_OPTIONS) errors_options[:file_size] = help.number_to_human_size check_value diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 7d3aa532750..8cb4060cd97 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -101,7 +101,7 @@ module Gitlab if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included # in the Service.available_services_names whitelist. - service = project.public_send("#{underscored_service}_service") + service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend if service && service.activated? && service.valid_token?(password) Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) @@ -149,7 +149,7 @@ module Gitlab def abilities_for_scope(scopes) scopes.map do |scope| - self.public_send(:"#{scope}_scope_authentication_abilities") + self.public_send(:"#{scope}_scope_authentication_abilities") # rubocop:disable GitlabSecurity/PublicSend end.flatten.uniq end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index f1a04affd38..754a45c3257 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -69,7 +69,7 @@ module Gitlab instance_variable_set(ivar_name, {}) end - key = __send__(cache_key_method_name, args) + key = __send__(cache_key_method_name, args) # rubocop:disable GitlabSecurity/PublicSend store.fetch(key) { store[key] = super(*args) } end diff --git a/lib/gitlab/diff/line_mapper.rb b/lib/gitlab/diff/line_mapper.rb index 576a761423e..cf71d47df8e 100644 --- a/lib/gitlab/diff/line_mapper.rb +++ b/lib/gitlab/diff/line_mapper.rb @@ -38,7 +38,7 @@ module Gitlab # - The first diff line with a higher line number, if it falls between diff contexts # - The last known diff line, if it falls after the last diff context diff_line = diff_lines.find do |diff_line| - diff_from_line = diff_line.send(from) + diff_from_line = diff_line.public_send(from) # rubocop:disable GitlabSecurity/PublicSend diff_from_line && diff_from_line >= from_line end diff_line ||= diff_lines.last @@ -47,8 +47,8 @@ module Gitlab # mapped line number is the same as the specified line number. return from_line unless diff_line - diff_from_line = diff_line.send(from) - diff_to_line = diff_line.send(to) + diff_from_line = diff_line.public_send(from) # rubocop:disable GitlabSecurity/PublicSend + diff_to_line = diff_line.public_send(to) # rubocop:disable GitlabSecurity/PublicSend # If the line was removed, there is no mapped line number. return unless diff_to_line diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 77b81d2d437..59e95191464 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -173,7 +173,7 @@ module Gitlab def initialize(options) %w(id name path size data mode commit_id binary).each do |key| - self.send("#{key}=", options[key.to_sym]) + self.__send__("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end @loaded_all_data = false diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 8e959c57c7c..b54962a4456 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -89,7 +89,7 @@ module Gitlab def initialize(options) %w(id root_id name path type mode commit_id).each do |key| - self.send("#{key}=", options[key.to_sym]) + self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 70177cd0fec..9a5f4f598b2 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -55,7 +55,7 @@ module Gitlab def self.call(storage, service, rpc, request) metadata = request_metadata(storage) metadata = yield(metadata) if block_given? - stub(service, storage).send(rpc, request, metadata) + stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend end def self.request_metadata(storage) diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 8c80791e7c9..f330041cc00 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -11,7 +11,9 @@ module Gitlab end def create! - project.public_send(project_association).find_or_create_by!(find_condition) do |record| + association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend + + association.find_or_create_by!(find_condition) do |record| record.attributes = attributes end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 7dbeec5b010..0550f9695bd 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -120,7 +120,7 @@ module Gitlab def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = api.send(method, *args) + data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend return data unless data.is_a?(Array) last_response = api.last_response diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 266b1a6fece..373062b354b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -289,7 +289,7 @@ module Gitlab opts.last[:page] = current_page(resource_type) - client.public_send(resource_type, *opts) do |resources| + client.public_send(resource_type, *opts) do |resources| # rubocop:disable GitlabSecurity/PublicSend yield resources increment_page(resource_type) end diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb index 2a659ae4c74..99594577141 100644 --- a/lib/gitlab/lazy.rb +++ b/lib/gitlab/lazy.rb @@ -16,7 +16,7 @@ module Gitlab def method_missing(name, *args, &block) __evaluate__ - @result.__send__(name, *args, &block) + @result.__send__(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(name, include_private = false) diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 43eb73250b7..e138b466a34 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -32,7 +32,7 @@ module Gitlab end def uid - entry.send(config.uid).first + entry.public_send(config.uid).first # rubocop:disable GitlabSecurity/PublicSend end def username @@ -65,7 +65,7 @@ module Gitlab return nil unless selected_attr - entry.public_send(selected_attr) + entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb index 699d8b9fc07..306923902e0 100644 --- a/lib/gitlab/markdown/pipeline.rb +++ b/lib/gitlab/markdown/pipeline.rb @@ -23,7 +23,7 @@ module Gitlab define_method(meth) do |text, context| context = transform_context(context) - html_pipeline.send(meth, text, context) + html_pipeline.__send__(meth, text, context) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 41dee5fdc06..4a3c40f88eb 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -27,7 +27,7 @@ class UploadedFile alias_method :local_path, :path def method_missing(method_name, *args, &block) #:nodoc: - @tempfile.__send__(method_name, *args, &block) + @tempfile.__send__(method_name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end def respond_to?(method_name, include_private = false) #:nodoc: -- cgit v1.2.1 From e99444bb2d3a249461825550fc6271f4e0ee8874 Mon Sep 17 00:00:00 2001 From: vanadium23 Date: Mon, 7 Aug 2017 21:00:11 +0300 Subject: Fix CI_PROJECT_PATH_SLUG slugify --- lib/gitlab/utils.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index fa182c4deda..9670c93759e 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -14,6 +14,19 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + # A slugified version of the string, suitable for inclusion in URLs and + # domain names. Rules: + # + # * Lowercased + # * Anything not matching [a-z0-9-] is replaced with a - + # * Maximum length is 63 bytes + # * First/Last Character is not a hyphen + def slugify(str) + return str.downcase + .gsub(/[^a-z0-9]/, '-')[0..62] + .gsub(/(\A-+|-+\z)/, '') + end + def to_boolean(value) return value if [true, false].include?(value) return true if value =~ /^(true|t|yes|y|1|on)$/i -- cgit v1.2.1 From 639140677410f11955947980aa47356e1136c32e Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" Date: Wed, 9 Aug 2017 12:16:43 +0200 Subject: Add two more project templates Related to !13108. Mostly this is just running the rake task and changing the task a bit to catch cases like the project already existing or so. The rake task moves archives to the vendor/project_template directory, which are checked in too. --- lib/gitlab/project_template.rb | 4 +++- lib/tasks/gitlab/update_templates.rake | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index cf461adf697..732fbf68dad 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -25,7 +25,9 @@ module Gitlab end TEMPLATES_TABLE = [ - ProjectTemplate.new('rails', 'Ruby on Rails') + ProjectTemplate.new('rails', 'Ruby on Rails'), + ProjectTemplate.new('spring', 'Spring'), + ProjectTemplate.new('express', 'NodeJS Express') ].freeze class << self diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index a7e30423c7a..f44abc2b81b 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -21,13 +21,18 @@ namespace :gitlab do params = { import_url: template.clone_url, namespace_id: admin.namespace.id, - path: template.title, + path: template.name, skip_wiki: true } - puts "Creating project for #{template.name}" + puts "Creating project for #{template.title}" project = Projects::CreateService.new(admin, params).execute + unless project.persisted? + puts project.errors.messages + exit(1) + end + loop do if project.finished? puts "Import finished for #{template.name}" -- cgit v1.2.1 From b7e98620046501144302ff8f9fbf22ff03ef4db7 Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Thu, 10 Aug 2017 16:16:54 -0300 Subject: Fix API responses when dealing with txt files --- lib/api/files.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/api/files.rb b/lib/api/files.rb index 450334fee84..e2ac7142bc4 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,5 +1,8 @@ module API class Files < Grape::API + # Prevents returning plain/text responses for files with .txt extension + after_validation { content_type "application/json" } + helpers do def commit_params(attrs) { -- cgit v1.2.1 From 4405c8e393c8621f1a5f3b31f6e7578419562716 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 15 Aug 2017 16:39:31 +0200 Subject: Remove deprecated field from workhorse API responses --- lib/gitlab/workhorse.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 3f25e463412..a362a3a0bc6 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -45,7 +45,6 @@ module Gitlab raise "Unsupported action: #{action}" end if feature_enabled - params[:GitalyAddress] = server[:address] # This field will be deprecated params[:GitalyServer] = server end -- cgit v1.2.1 From 5fc871381ad0768bb38879ab1621e538ca3008d0 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Mon, 14 Aug 2017 12:29:47 +0100 Subject: Speed up project creation by inlining repository creation --- lib/gitlab/git/repository.rb | 22 ++++++++++++++++++++++ lib/gitlab/shell.rb | 12 +++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 38772d06dbd..1d5ca68137a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -18,6 +18,28 @@ module Gitlab InvalidBlobName = Class.new(StandardError) InvalidRef = Class.new(StandardError) + class << self + # Unlike `new`, `create` takes the storage path, not the storage name + def create(storage_path, name, bare: true, symlink_hooks_to: nil) + repo_path = File.join(storage_path, name) + repo_path += '.git' unless repo_path.end_with?('.git') + + FileUtils.mkdir_p(repo_path, mode: 0770) + + # Equivalent to `git --git-path=#{repo_path} init [--bare]` + repo = Rugged::Repository.init_at(repo_path, bare) + repo.close + + if symlink_hooks_to.present? + hooks_path = File.join(repo_path, 'hooks') + FileUtils.rm_rf(hooks_path) + FileUtils.ln_s(symlink_hooks_to, hooks_path) + end + + true + end + end + # Full path to repo attr_reader :path diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 0cb28732402..280a9abf03e 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -73,8 +73,10 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) - gitlab_shell_fast_execute([gitlab_shell_projects_path, - 'add-project', storage, "#{name}.git"]) + Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + rescue => err + Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") + false end # Import repository @@ -273,7 +275,11 @@ module Gitlab protected def gitlab_shell_path - Gitlab.config.gitlab_shell.path + File.expand_path(Gitlab.config.gitlab_shell.path) + end + + def gitlab_shell_hooks_path + File.expand_path(Gitlab.config.gitlab_shell.hooks_path) end def gitlab_shell_user_home -- cgit v1.2.1 From 4edfad96784e8f77ec8ead26f01b4012977ba58a Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 15 Aug 2017 13:44:37 -0400 Subject: Enable Layout/TrailingWhitespace cop and auto-correct offenses --- lib/api/entities.rb | 2 +- lib/api/protected_branches.rb | 2 +- lib/banzai/filter/image_lazy_load_filter.rb | 4 ++-- lib/constraints/project_url_constrainer.rb | 2 +- lib/declarative_policy/base.rb | 2 +- lib/gitlab/auth/ip_rate_limiter.rb | 12 ++++++------ lib/gitlab/ci/build/artifacts/metadata.rb | 2 +- lib/gitlab/git/blob.rb | 2 +- lib/gitlab/import_export/attributes_finder.rb | 2 +- lib/gitlab/ldap/auth_hash.rb | 2 +- lib/gitlab/middleware/rails_queue_duration.rb | 2 +- lib/gitlab/slash_commands/presenters/help.rb | 2 +- lib/tasks/gitlab/gitaly.rake | 2 +- 13 files changed, 19 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 18cd604a216..4219808fdc3 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -83,7 +83,7 @@ module API expose :created_at, :last_activity_at end - class Project < BasicProjectDetails + class Project < BasicProjectDetails include ::API::Helpers::RelatedResourcesHelpers expose :_links do diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index d742f2e18d0..dccf4fa27a7 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -61,7 +61,7 @@ module API service_args = [user_project, current_user, protected_branch_params] protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute - + if protected_branch.persisted? present protected_branch, with: Entities::ProtectedBranch, project: user_project else diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb index 7a81d583b82..bcb4f332267 100644 --- a/lib/banzai/filter/image_lazy_load_filter.rb +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -6,9 +6,9 @@ module Banzai doc.xpath('descendant-or-self::img').each do |img| img['class'] ||= '' << 'lazy' img['data-src'] = img['src'] - img['src'] = LazyImageTagHelper.placeholder_image + img['src'] = LazyImageTagHelper.placeholder_image end - + doc end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index fd7b97d3167..5bef29eb1da 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -7,7 +7,7 @@ class ProjectUrlConstrainer return false unless DynamicPathValidator.valid_project_path?(full_path) # We intentionally allow SELECT(*) here so result of this query can be used - # as cache for further Project.find_by_full_path calls within request + # as cache for further Project.find_by_full_path calls within request Project.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index df94cafb6a1..f39b5bf29ec 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -221,7 +221,7 @@ module DeclarativePolicy end # computes the given ability and prints a helpful debugging output - # showing which + # showing which def debug(ability, *a) runner(ability).debug(*a) end diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb index 1089bc9f89e..e6173d45af3 100644 --- a/lib/gitlab/auth/ip_rate_limiter.rb +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -11,11 +11,11 @@ module Gitlab def enabled? config.enabled end - + def reset! Rack::Attack::Allow2Ban.reset(ip, config) end - + def register_fail! # Allow2Ban.filter will return false if this IP has not failed too often yet @banned = Rack::Attack::Allow2Ban.filter(ip, config) do @@ -23,17 +23,17 @@ module Gitlab ip_can_be_banned? end end - + def banned? @banned end - + private - + def config Gitlab.config.rack_attack.git_basic_auth end - + def ip_can_be_banned? config.ip_whitelist.exclude?(ip) end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index a375ccbece0..a788fb3fcbc 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -60,7 +60,7 @@ module Gitlab begin path = read_string(gz).force_encoding('UTF-8') meta = read_string(gz).force_encoding('UTF-8') - + next unless path.valid_encoding? && meta.valid_encoding? next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 77b81d2d437..28835d7f5d2 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -54,7 +54,7 @@ module Gitlab # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the # full blob contents are returned. If blob_size_limit >= 0 then each blob will # contain no more than limit bytes in its data attribute. - # + # # Keep in mind that this method may allocate a lot of memory. It is up # to the caller to limit the number of blobs and blob_size_limit. # diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index d230de781d5..c4fa91ef8d6 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -1,7 +1,7 @@ module Gitlab module ImportExport class AttributesFinder - + def initialize(included_attributes:, excluded_attributes:, methods:) @included_attributes = included_attributes || {} @excluded_attributes = excluded_attributes || {} diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 95378e5a769..4fbc5fa5262 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -17,7 +17,7 @@ module Gitlab value = value.first if value break if value.present? end - + return super unless value Gitlab::Utils.force_utf8(value) diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 5d2d7d0026c..63c3372da51 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -8,7 +8,7 @@ module Gitlab def initialize(app) @app = app end - + def call(env) trans = Gitlab::Metrics.current_transaction proxy_start = env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index ea611a4d629..ab855319077 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -14,7 +14,7 @@ module Gitlab if text.start_with?('help') header_with_list("Available commands", full_commands(trigger)) else - header_with_list("Unknown command, these commands are available", full_commands(trigger)) + header_with_list("Unknown command, these commands are available", full_commands(trigger)) end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index e337c67a0f5..08677a98fc1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -18,7 +18,7 @@ namespace :gitlab do command = status.zero? ? ['gmake'] : ['make'] if Rails.env.test? - command += %W[BUNDLE_PATH=#{Bundler.bundle_path}] + command += %W[BUNDLE_PATH=#{Bundler.bundle_path}] end Dir.chdir(args.dir) do -- cgit v1.2.1 From f5cb3ac14da14a527a9a259d8a1bfb372868f352 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 15 Aug 2017 15:08:56 -0400 Subject: Don't depend on `Rails` for Redis configuration file paths The `Rails` object was not always available in all tasks that require Redis access, such as `mail_room`, so the constant pointing to the configuration path was never defined, but we still attempted to access it in `config_file_name`, resulting in a `NameError` exception. Further, there was no benefit to defining these paths in a constant to begin with -- they're only accessed in one place, and it was within the class where they were being defined. We can just provide them at run-time instead. Further _still_, we were calling `File.expand_path` on the absolute path returned by `Rails.root.join`, which was rather pointless. --- lib/gitlab/redis/cache.rb | 5 +---- lib/gitlab/redis/queues.rb | 5 +---- lib/gitlab/redis/shared_state.rb | 5 +---- lib/gitlab/redis/wrapper.rb | 13 +++++++++---- 4 files changed, 12 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index b0da516ff83..9bf019b72e6 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -7,9 +7,6 @@ module Gitlab CACHE_NAMESPACE = 'cache:gitlab'.freeze DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze - if defined?(::Rails) && ::Rails.root.present? - DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze - end class << self def default_url @@ -22,7 +19,7 @@ module Gitlab return file_name unless file_name.nil? # otherwise, if config files exists for this class, use it - file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + file_name = config_file_path('redis.cache.yml') return file_name if File.file?(file_name) # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index f9249d05565..e1695aafbeb 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -8,9 +8,6 @@ module Gitlab MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze - if defined?(::Rails) && ::Rails.root.present? - DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze - end class << self def default_url @@ -23,7 +20,7 @@ module Gitlab return file_name if file_name # otherwise, if config files exists for this class, use it - file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + file_name = config_file_path('redis.queues.yml') return file_name if File.file?(file_name) # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 395dcf082da..10bec7a90da 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -7,9 +7,6 @@ module Gitlab SESSION_NAMESPACE = 'session:gitlab'.freeze DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze - if defined?(::Rails) && ::Rails.root.present? - DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze - end class << self def default_url @@ -22,7 +19,7 @@ module Gitlab return file_name if file_name # otherwise, if config files exists for this class, use it - file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + file_name = config_file_path('redis.shared_state.yml') return file_name if File.file?(file_name) # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index c43b37dde74..8ad06480575 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -8,9 +8,6 @@ module Gitlab class Wrapper DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze - if defined?(::Rails) && ::Rails.root.present? - DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze - end class << self delegate :params, :url, to: :new @@ -49,13 +46,21 @@ module Gitlab DEFAULT_REDIS_URL end + # Return the absolute path to a Rails configuration file + # + # We use this instead of `Rails.root` because for certain tasks + # utilizing these classes, `Rails` might not be available. + def config_file_path(filename) + File.expand_path("../../../config/#{filename}", __dir__) + end + def config_file_name # if ENV set for wrapper class, use it even if it points to a file does not exist file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] return file_name unless file_name.nil? # otherwise, if config files exists for wrapper class, use it - file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + file_name = config_file_path('resque.yml') return file_name if File.file?(file_name) # nil will force use of DEFAULT_REDIS_URL when config file is absent -- cgit v1.2.1 From 142b9ec4a0b3e5d5cbcbb2fc3b201755f0966b9f Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 15 Aug 2017 15:53:16 -0400 Subject: Fix two additional violations caused by previous changes --- lib/gitlab/import_export/attributes_finder.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index c4fa91ef8d6..56042ddecbf 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class AttributesFinder - def initialize(included_attributes:, excluded_attributes:, methods:) @included_attributes = included_attributes || {} @excluded_attributes = excluded_attributes || {} -- cgit v1.2.1 From ba7251fefd92b0ecb6365cfe55510e24c5343ac6 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 15 Aug 2017 13:22:55 +0200 Subject: Only create commit GPG signature when necessary --- lib/gitlab/git/commit.rb | 19 +++++++------ lib/gitlab/gpg/commit.rb | 38 ++++++++++++++++--------- lib/gitlab/gpg/invalid_gpg_signature_updater.rb | 4 +-- 3 files changed, 36 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index fd4dfdb09a2..a499bbc6266 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -210,6 +210,16 @@ module Gitlab @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) end + + def shas_with_signatures(repository, shas) + shas.select do |sha| + begin + Rugged::Commit.extract_signature(repository.rugged, sha) + rescue Rugged::OdbError + false + end + end + end end def initialize(repository, raw_commit, head = nil) @@ -335,15 +345,6 @@ module Gitlab parent_ids.map { |oid| self.class.find(@repository, oid) }.compact end - # Get the gpg signature of this commit. - # - # Ex. - # commit.signature(repo) - # - def signature(repo) - Rugged::Commit.extract_signature(repo.rugged, sha) - end - def stats Gitlab::Git::CommitStats.new(self) end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 55428b85207..606c7576f70 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -1,12 +1,20 @@ module Gitlab module Gpg class Commit - attr_reader :commit + def self.for_commit(commit) + new(commit.project, commit.sha) + end - def initialize(commit) - @commit = commit + def initialize(project, sha) + @project = project + @sha = sha - @signature_text, @signed_text = commit.raw.signature(commit.project.repository) + @signature_text, @signed_text = + begin + Rugged::Commit.extract_signature(project.repository.rugged, sha) + rescue Rugged::OdbError + nil + end end def has_signature? @@ -16,18 +24,20 @@ module Gitlab def signature return unless has_signature? - cached_signature = GpgSignature.find_by(commit_sha: commit.sha) - return cached_signature if cached_signature.present? + return @signature if @signature - using_keychain do |gpg_key| - create_cached_signature!(gpg_key) - end + cached_signature = GpgSignature.find_by(commit_sha: @sha) + return @signature = cached_signature if cached_signature.present? + + @signature = create_cached_signature! end def update_signature!(cached_signature) using_keychain do |gpg_key| cached_signature.update_attributes!(attributes(gpg_key)) end + + @signature = cached_signature end private @@ -55,16 +65,18 @@ module Gitlab end end - def create_cached_signature!(gpg_key) - GpgSignature.create!(attributes(gpg_key)) + def create_cached_signature! + using_keychain do |gpg_key| + GpgSignature.create!(attributes(gpg_key)) + end end def attributes(gpg_key) user_infos = user_infos(gpg_key) { - commit_sha: commit.sha, - project: commit.project, + commit_sha: @sha, + project: @project, gpg_key: gpg_key, gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index 3bb491120ba..a525ee7a9ee 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -10,9 +10,7 @@ module Gitlab .select(:id, :commit_sha, :project_id) .where('gpg_key_id IS NULL OR valid_signature = ?', false) .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) - .find_each do |gpg_signature| - Gitlab::Gpg::Commit.new(gpg_signature.commit).update_signature!(gpg_signature) - end + .find_each { |sig| sig.gpg_commit.update_signature!(sig) } end end end -- cgit v1.2.1 From dc8e1676cda528cfca9ec9679da10ff90ec76282 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 16 Aug 2017 18:06:59 +0200 Subject: Upgrade grape to 1.0 Main feature was the deprication of the Hashie stuff, so the access by calling keys as method is gone now. --- lib/api/helpers.rb | 2 +- lib/api/jobs.rb | 4 ++-- lib/api/templates.rb | 6 +++--- lib/api/v3/builds.rb | 2 +- lib/api/v3/templates.rb | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3582ed81b0f..b56fd2388b3 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -290,7 +290,7 @@ module API def uploaded_file(field, uploads_path) if params[field] - bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) + bad_request!("#{field} is not a file") unless params[field][:filename] return params[field] end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 8a67de10bca..a40018b214e 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -16,9 +16,9 @@ module API case scope when String [scope] - when Hashie::Mash + when ::Hash scope.values - when Hashie::Array + when ::Array scope else ['unknown'] diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 0fc13b35d5b..f70bc0622b7 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -57,7 +57,7 @@ module API end get "templates/licenses" do options = { - featured: declared(params).popular.present? ? true : nil + featured: declared(params)[:popular].present? ? true : nil } licences = ::Kaminari.paginate_array(Licensee::License.all(options)) present paginate(licences), with: Entities::RepoLicense @@ -71,7 +71,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do - not_found!('License') unless Licensee::License.find(declared(params).name) + not_found!('License') unless Licensee::License.find(declared(params)[:name]) template = parsed_license_template @@ -102,7 +102,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get "templates/#{template_type}/:name" do - new_template = klass.find(declared(params).name) + new_template = klass.find(declared(params)[:name]) render_response(template_type, new_template) end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 93ad9eb26b8..c189d486f50 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -16,7 +16,7 @@ module API coerce_with: ->(scope) { if scope.is_a?(String) [scope] - elsif scope.is_a?(Hashie::Mash) + elsif scope.is_a?(::Hash) scope.values else ['unknown'] diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb index 4c577a8d2b7..2a2fb59045c 100644 --- a/lib/api/v3/templates.rb +++ b/lib/api/v3/templates.rb @@ -59,7 +59,7 @@ module API end get route do options = { - featured: declared(params).popular.present? ? true : nil + featured: declared(params)[:popular].present? ? true : nil } present Licensee::License.all(options), with: ::API::Entities::RepoLicense end @@ -76,7 +76,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get route, requirements: { name: /[\w\.-]+/ } do - not_found!('License') unless Licensee::License.find(declared(params).name) + not_found!('License') unless Licensee::License.find(declared(params)[:name]) template = parsed_license_template @@ -111,7 +111,7 @@ module API requires :name, type: String, desc: 'The name of the template' end get route do - new_template = klass.find(declared(params).name) + new_template = klass.find(declared(params)[:name]) render_response(template_type, new_template) end -- cgit v1.2.1 From 91035871d3f8ac39831a740e1b4c3865f2a87ea3 Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Thu, 17 Aug 2017 10:40:19 +0000 Subject: Fix project import to group when there are project milestones --- lib/gitlab/import_export/project_tree_restorer.rb | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 84ab1977dfa..cbc8d170936 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,6 +1,9 @@ module Gitlab module ImportExport class ProjectTreeRestorer + # Relations which cannot have both group_id and project_id at the same time + RESTRICT_PROJECT_AND_GROUP = %i(milestones).freeze + def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user @@ -118,9 +121,11 @@ module Gitlab end def create_relation(relation, relation_hash_list) + relation_type = relation.to_sym + relation_array = [relation_hash_list].flatten.map do |relation_hash| - Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: parsed_relation_hash(relation_hash), + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation_type, + relation_hash: parsed_relation_hash(relation_hash, relation_type), members_mapper: members_mapper, user: @user, project: restored_project) @@ -129,8 +134,16 @@ module Gitlab relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end - def parsed_relation_hash(relation_hash) - relation_hash.merge!('group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id) + def parsed_relation_hash(relation_hash, relation_type) + if RESTRICT_PROJECT_AND_GROUP.include?(relation_type) + params = {} + params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id'] + params['project_id'] = restored_project.id if relation_hash['project_id'] + else + params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id } + end + + relation_hash.merge(params) end end end -- cgit v1.2.1 From 40752ce7e98b7bab03ffa9d49ea8a26f5d0e098b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 16 Aug 2017 14:29:27 +0200 Subject: Use grpc 1.4.5 --- lib/gitlab/gitaly_client/blob_service.rb | 13 ++++++++++--- lib/gitlab/gitaly_client/commit_service.rb | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 7ea8e8d0857..a250eb75bd4 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -13,10 +13,17 @@ module Gitlab ) response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request) - blob = response.first - return unless blob.oid.present? + data = '' + blob = nil + response.each do |msg| + if blob.nil? + blob = msg + end - data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup } + data << msg.data + end + + return nil if blob.oid.blank? Gitlab::Git::Blob.new( id: blob.oid, diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 93268d9f33c..b36e81278d6 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -60,15 +60,21 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) - entry = response.first - return unless entry.oid.present? - if entry.type == :BLOB - rest_of_data = response.reduce("") { |memo, msg| memo << msg.data } - entry.data += rest_of_data + entry = nil + data = '' + response.each do |msg| + if entry.nil? + entry = msg + + break unless entry.type == :BLOB + end + + data << msg.data end + entry.data = data - entry + entry unless entry.oid.blank? end def tree_entries(repository, revision, path) -- cgit v1.2.1 From f865b1b459b7c53eb57580246d8e20c4fd45f7fe Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 17 Aug 2017 10:00:31 +0100 Subject: Backports EE mirror stuck handling feature (https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2628) to CE --- lib/after_commit_queue.rb | 30 ++++++++++++++++++++++++++++++ lib/gitlab/sidekiq_status.rb | 9 +++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 lib/after_commit_queue.rb (limited to 'lib') diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb new file mode 100644 index 00000000000..b67575a3ac2 --- /dev/null +++ b/lib/after_commit_queue.rb @@ -0,0 +1,30 @@ +module AfterCommitQueue + extend ActiveSupport::Concern + + included do + after_commit :_run_after_commit_queue + after_rollback :_clear_after_commit_queue + end + + def run_after_commit(method = nil, &block) + _after_commit_queue << proc { self.send(method) } if method + _after_commit_queue << block if block + true + end + + protected + + def _run_after_commit_queue + while action = _after_commit_queue.pop + self.instance_eval(&action) + end + end + + def _after_commit_queue + @after_commit_queue ||= [] + end + + def _clear_after_commit_queue + _after_commit_queue.clear + end +end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index ca8d3271541..a0a2769cf9e 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -90,9 +90,14 @@ module Gitlab # # Returns an array of completed JIDs def self.completed_jids(job_ids) - Sidekiq.redis do |redis| - job_ids.reject { |jid| redis.exists(key_for(jid)) } + statuses = job_status(job_ids) + + completed = [] + job_ids.zip(statuses).each do |job_id, status| + completed << job_id unless status end + + completed end def self.key_for(jid) -- cgit v1.2.1 From c463a834058b5cede382334932891cf3e4cdb35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Fri, 11 Aug 2017 18:44:25 -0400 Subject: Incorporate RepositoryService.ApplyGitattributes Gitaly RPC --- lib/gitlab/git/repository.rb | 68 ++++++++++++++++---------- lib/gitlab/gitaly_client/repository_service.rb | 5 ++ 2 files changed, 47 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1d5ca68137a..aef7ae659fe 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -653,33 +653,15 @@ module Gitlab # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) - begin - commit = lookup(ref) - rescue Rugged::ReferenceError - raise InvalidRef.new("Ref #{ref} is invalid") - end - - # Create the paths - info_dir_path = File.join(path, 'info') - info_attributes_path = File.join(info_dir_path, 'attributes') - - begin - # Retrieve the contents of the blob - gitattributes_content = blob_content(commit, '.gitattributes') - rescue InvalidBlobName - # No .gitattributes found. Should now remove any info/attributes and return - File.delete(info_attributes_path) if File.exist?(info_attributes_path) - return - end - - # Create the info directory if needed - Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) - - # Write the contents of the .gitattributes file to info/attributes - # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 - File.open(info_attributes_path, "wb") do |file| - file.write(gitattributes_content) + Gitlab::GitalyClient.migrate(:apply_gitattributes) do |is_enabled| + if is_enabled + gitaly_copy_gitattributes(ref) + else + rugged_copy_gitattributes(ref) + end end + rescue GRPC::InvalidArgument + raise InvalidRef end # Returns the Git attributes for the given file path. @@ -1012,6 +994,40 @@ module Gitlab raw_output.compact end + + def gitaly_copy_gitattributes(revision) + gitaly_repository_client.apply_gitattributes(revision) + end + + def rugged_copy_gitattributes(ref) + begin + commit = lookup(ref) + rescue Rugged::ReferenceError + raise InvalidRef.new("Ref #{ref} is invalid") + end + + # Create the paths + info_dir_path = File.join(path, 'info') + info_attributes_path = File.join(info_dir_path, 'attributes') + + begin + # Retrieve the contents of the blob + gitattributes_content = blob_content(commit, '.gitattributes') + rescue InvalidBlobName + # No .gitattributes found. Should now remove any info/attributes and return + File.delete(info_attributes_path) if File.exist?(info_attributes_path) + return + end + + # Create the info directory if needed + Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) + + # Write the contents of the .gitattributes file to info/attributes + # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 + File.open(info_attributes_path, "wb") do |file| + file.write(gitattributes_content) + end + end end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 6ad97e62941..a74a6dc6e78 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -32,6 +32,11 @@ module Gitlab request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) GitalyClient.call(@storage, :repository_service, :repository_size, request).size end + + def apply_gitattributes(revision) + request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: revision) + GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) + end end end end -- cgit v1.2.1 From c271fdc80fd17e1931a1d912e231bec2b8e3d098 Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Tue, 15 Aug 2017 07:09:05 +0200 Subject: Don't escape html entities when rich == raw line --- lib/gitlab/string_range_marker.rb | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 94fba0a221a..11aeec1ebfa 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -1,21 +1,31 @@ module Gitlab class StringRangeMarker - attr_accessor :raw_line, :rich_line - - def initialize(raw_line, rich_line = raw_line) - @raw_line = raw_line - @rich_line = ERB::Util.html_escape(rich_line) + attr_accessor :raw_line, :rich_line, :html_escaped + + def initialize(raw_line, rich_line = nil) + @raw_line = raw_line.dup + if rich_line.nil? + @rich_line = raw_line.dup + @html_escaped = false + else + @rich_line = ERB::Util.html_escape(rich_line) + @html_escaped = true + end end def mark(marker_ranges) return rich_line unless marker_ranges - rich_marker_ranges = [] - marker_ranges.each do |range| - # Map the inline-diff range based on the raw line to character positions in the rich line - rich_positions = position_mapping[range].flatten - # Turn the array of character positions into ranges - rich_marker_ranges.concat(collapse_ranges(rich_positions)) + if html_escaped + rich_marker_ranges = [] + marker_ranges.each do |range| + # Map the inline-diff range based on the raw line to character positions in the rich line + rich_positions = position_mapping[range].flatten + # Turn the array of character positions into ranges + rich_marker_ranges.concat(collapse_ranges(rich_positions)) + end + else + rich_marker_ranges = marker_ranges end offset = 0 @@ -31,7 +41,7 @@ module Gitlab offset += text.length - original_text.length end - rich_line.html_safe + @html_escaped ? rich_line.html_safe : rich_line end private -- cgit v1.2.1 From fdf4f0fc0884c0346b16ec107a1ea1084dd4a32f Mon Sep 17 00:00:00 2001 From: Jordan Patterson Date: Thu, 17 Aug 2017 09:18:07 -0300 Subject: don't add next page link if current page is out of range --- lib/api/helpers/pagination.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 0764b58fb4c..e86e221e092 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -30,7 +30,7 @@ module API links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? request_params[:page] = paginated_data.current_page + 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? || paginated_data.out_of_range? request_params[:page] = 1 links << %(<#{request_url}?#{request_params.to_query}>; rel="first") -- cgit v1.2.1 From e17d9529faa4d292d7f869a0f9ebbbcde4cc6f9e Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Thu, 17 Aug 2017 17:56:01 +0200 Subject: Total Pages should be at least one And the link to the last page cannot be `page=0`. --- lib/api/helpers/pagination.rb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index e86e221e092..95108292aac 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -11,7 +11,7 @@ module API def add_pagination_headers(paginated_data) header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Total-Pages', total_pages(paginated_data).to_s header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s @@ -26,20 +26,25 @@ module API links = [] - request_params[:page] = paginated_data.current_page - 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + request_params[:page] = paginated_data.prev_page + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") if request_params[:page] - request_params[:page] = paginated_data.current_page + 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? || paginated_data.out_of_range? + request_params[:page] = paginated_data.next_page + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") if request_params[:page] request_params[:page] = 1 links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - request_params[:page] = paginated_data.total_pages + request_params[:page] = total_pages(paginated_data) links << %(<#{request_url}?#{request_params.to_query}>; rel="last") links.join(', ') end + + def total_pages(paginated_data) + # Ensure there is in total at least 1 page + [paginated_data.total_pages, 1].max + end end end end -- cgit v1.2.1 From aaa887febad442ecdb54680954006e63af70ac6c Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Mon, 14 Aug 2017 12:12:32 +0100 Subject: Client Implementation: RefService::RefExists --- lib/gitlab/git/repository.rb | 51 +++++++++++++++++++++++++++------ lib/gitlab/gitaly_client/ref_service.rb | 8 ++++++ 2 files changed, 50 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index aef7ae659fe..53aa5b12489 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -204,21 +204,26 @@ module Gitlab # # name - The name of the tag as a String. def tag_exists?(name) - !!rugged.tags[name] + gitaly_migrate(:ref_exists_tags) do |is_enabled| + if is_enabled + gitaly_ref_exists?("refs/tags/#{name}") + else + rugged_tag_exists?(name) + end + end end # Returns true if the given branch exists # # name - The name of the branch as a String. def branch_exists?(name) - rugged.branches.exists?(name) - - # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. - # Whatever code calls this method shouldn't have to deal with that so - # instead we just return `false` (which is true since a branch doesn't - # exist when it has an invalid name). - rescue Rugged::ReferenceError - false + gitaly_migrate(:ref_exists_branches) do |is_enabled| + if is_enabled + gitaly_ref_exists?("refs/heads/#{name}") + else + rugged_branch_exists?(name) + end + end end # Returns an Array of branch and tag names @@ -995,6 +1000,34 @@ module Gitlab raw_output.compact end + # Returns true if the given ref name exists + # + # Ref names must start with `refs/`. + def gitaly_ref_exists?(ref_name) + gitaly_ref_client.ref_exists?(ref_name) + end + + # Returns true if the given tag exists + # + # name - The name of the tag as a String. + def rugged_tag_exists?(name) + !!rugged.tags[name] + end + + # Returns true if the given branch exists + # + # name - The name of the branch as a String. + def rugged_branch_exists?(name) + rugged.branches.exists?(name) + + # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. + # Whatever code calls this method shouldn't have to deal with that so + # instead we just return `false` (which is true since a branch doesn't + # exist when it has an invalid name). + rescue Rugged::ReferenceError + false + end + def gitaly_copy_gitattributes(revision) gitaly_repository_client.apply_gitattributes(revision) end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 919fb68b8c7..cdcfed36740 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -70,6 +70,14 @@ module Gitlab consume_tags_response(response) end + def ref_exists?(ref_name) + request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: ref_name) + response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) + response.value + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message + end + private def consume_refs_response(response) -- cgit v1.2.1 From 1ffd0c8562922de0e70eef2faf71108e0eecbb51 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Fri, 18 Aug 2017 10:14:13 +0200 Subject: Remove CI API v1 This API was mainly for internal usage, and has been moved to the general API: APIv4. The endpoints have been deprecated since 9.0, and won't see 10.0. :) --- lib/ci/api/api.rb | 39 --------- lib/ci/api/builds.rb | 219 ------------------------------------------------- lib/ci/api/entities.rb | 93 --------------------- lib/ci/api/helpers.rb | 89 -------------------- lib/ci/api/runners.rb | 50 ----------- lib/ci/api/triggers.rb | 39 --------- 6 files changed, 529 deletions(-) delete mode 100644 lib/ci/api/api.rb delete mode 100644 lib/ci/api/builds.rb delete mode 100644 lib/ci/api/entities.rb delete mode 100644 lib/ci/api/helpers.rb delete mode 100644 lib/ci/api/runners.rb delete mode 100644 lib/ci/api/triggers.rb (limited to 'lib') diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb deleted file mode 100644 index 24bb3649a76..00000000000 --- a/lib/ci/api/api.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Ci - module API - class API < Grape::API - include ::API::APIGuard - version 'v1', using: :path - - rescue_from ActiveRecord::RecordNotFound do - rack_response({ 'message' => '404 Not found' }.to_json, 404) - end - - # Retain 405 error rather than a 500 error for Grape 0.15.0+. - # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes - rescue_from Grape::Exceptions::MethodNotAllowed do |e| - error! e.message, e.status, e.headers - end - - rescue_from Grape::Exceptions::Base do |e| - error! e.message, e.status, e.headers - end - - rescue_from :all do |exception| - handle_api_exception(exception) - end - - content_type :txt, 'text/plain' - content_type :json, 'application/json' - format :json - - helpers ::SentryHelper - helpers ::Ci::API::Helpers - helpers ::API::Helpers - helpers Gitlab::CurrentSettings - - mount ::Ci::API::Builds - mount ::Ci::API::Runners - mount ::Ci::API::Triggers - end - end -end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb deleted file mode 100644 index 79058c02ce5..00000000000 --- a/lib/ci/api/builds.rb +++ /dev/null @@ -1,219 +0,0 @@ -module Ci - module API - # Builds API - class Builds < Grape::API - resource :builds do - # Runs oldest pending build by runner - Runners only - # - # Parameters: - # token (required) - The uniq token of runner - # - # Example Request: - # POST /builds/register - post "register" do - authenticate_runner! - required_attributes! [:token] - not_found! unless current_runner.active? - update_runner_info - - if current_runner.is_runner_queue_value_latest?(params[:last_update]) - header 'X-GitLab-Last-Update', params[:last_update] - Gitlab::Metrics.add_event(:build_not_found_cached) - return build_not_found! - end - - new_update = current_runner.ensure_runner_queue_value - - result = Ci::RegisterJobService.new(current_runner).execute - - if result.valid? - if result.build - Gitlab::Metrics.add_event(:build_found, - project: result.build.project.full_path) - - present result.build, with: Entities::BuildDetails - else - Gitlab::Metrics.add_event(:build_not_found) - - header 'X-GitLab-Last-Update', new_update - - build_not_found! - end - else - # We received build that is invalid due to concurrency conflict - Gitlab::Metrics.add_event(:build_invalid) - conflict! - end - end - - # Update an existing build - Runners only - # - # Parameters: - # id (required) - The ID of a project - # state (optional) - The state of a build - # trace (optional) - The trace of a build - # Example Request: - # PUT /builds/:id - put ":id" do - authenticate_runner! - build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - validate_build!(build) - - update_runner_info - - build.trace.set(params[:trace]) if params[:trace] - - Gitlab::Metrics.add_event(:update_build, - project: build.project.full_path) - - case params[:state].to_s - when 'success' - build.success - when 'failed' - build.drop - end - end - - # Send incremental log update - Runners only - # - # Parameters: - # id (required) - The ID of a build - # Body: - # content of logs to append - # Headers: - # Content-Range (required) - range of content that was sent - # BUILD-TOKEN (required) - The build authorization token - # Example Request: - # PATCH /builds/:id/trace.txt - patch ":id/trace.txt" do - build = authenticate_build! - - error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') - content_range = request.headers['Content-Range'] - content_range = content_range.split('-') - - stream_size = build.trace.append(request.body.read, content_range[0].to_i) - if stream_size < 0 - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) - end - - status 202 - header 'Build-Status', build.status - header 'Range', "0-#{stream_size}" - end - - # Authorize artifacts uploading for build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # filesize (optional) - the size of uploaded file - # Example Request: - # POST /builds/:id/artifacts/authorize - post ":id/artifacts/authorize" do - require_gitlab_workhorse! - Gitlab::Workhorse.verify_api_request!(headers) - not_allowed! unless Gitlab.config.artifacts.enabled - build = authenticate_build! - forbidden!('build is not running') unless build.running? - - if params[:filesize] - file_size = params[:filesize].to_i - file_to_large! unless file_size < max_artifacts_size - end - - status 200 - content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - Gitlab::Workhorse.artifact_upload_ok - end - - # Upload artifacts to build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # file (required) - Artifacts file - # expire_in (optional) - Specify when artifacts should expire (ex. 7d) - # Parameters (accelerated by GitLab Workhorse): - # file.path - path to locally stored body (generated by Workhorse) - # file.name - real filename as send in Content-Disposition - # file.type - real content type as send in Content-Type - # metadata.path - path to locally stored body (generated by Workhorse) - # metadata.name - filename (generated by Workhorse) - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Body: - # The file content - # - # Example Request: - # POST /builds/:id/artifacts - post ":id/artifacts" do - require_gitlab_workhorse! - not_allowed! unless Gitlab.config.artifacts.enabled - build = authenticate_build! - forbidden!('Build is not running!') unless build.running? - - artifacts_upload_path = ArtifactUploader.artifacts_upload_path - artifacts = uploaded_file(:file, artifacts_upload_path) - metadata = uploaded_file(:metadata, artifacts_upload_path) - - bad_request!('Missing artifacts file!') unless artifacts - file_to_large! unless artifacts.size < max_artifacts_size - - build.artifacts_file = artifacts - build.artifacts_metadata = metadata - build.artifacts_expire_in = - params['expire_in'] || - Gitlab::CurrentSettings.current_application_settings - .default_artifacts_expire_in - - if build.save - present(build, with: Entities::BuildDetails) - else - render_validation_error!(build) - end - end - - # Download the artifacts file from build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Example Request: - # GET /builds/:id/artifacts - get ":id/artifacts" do - build = authenticate_build! - artifacts_file = build.artifacts_file - - unless artifacts_file.exists? - not_found! - end - - unless artifacts_file.file_storage? - return redirect_to build.artifacts_file.url - end - - present_file!(artifacts_file.path, artifacts_file.filename) - end - - # Remove the artifacts file from build - Runners only - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Headers: - # BUILD-TOKEN (required) - The build authorization token, the same as token - # Example Request: - # DELETE /builds/:id/artifacts - delete ":id/artifacts" do - build = authenticate_build! - - status(200) - build.erase_artifacts! - end - end - end - end -end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb deleted file mode 100644 index 31f66dd5a58..00000000000 --- a/lib/ci/api/entities.rb +++ /dev/null @@ -1,93 +0,0 @@ -module Ci - module API - module Entities - class Commit < Grape::Entity - expose :id, :sha, :project_id, :created_at - expose :status, :finished_at, :duration - expose :git_commit_message, :git_author_name, :git_author_email - end - - class CommitWithBuilds < Commit - expose :builds - end - - class ArtifactFile < Grape::Entity - expose :filename, :size - end - - class BuildOptions < Grape::Entity - expose :image - expose :services - expose :artifacts - expose :cache - expose :dependencies - expose :after_script - end - - class Build < Grape::Entity - expose :id, :ref, :tag, :sha, :status - expose :name, :token, :stage - expose :project_id - expose :project_name - expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } - end - - class BuildCredentials < Grape::Entity - expose :type, :url, :username, :password - end - - class BuildDetails < Build - expose :commands - expose :repo_url - expose :before_sha - expose :allow_git_fetch - expose :token - expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } - - expose :options do |model| - # This part ensures that output of old API is still the same after adding support - # for extended docker configuration options, used by new API - # - # I'm leaving this here, not in the model, because it should be removed at the same time - # when old API will be removed (planned for August 2017). - model.options.dup.tap do |options| - options[:image] = options[:image][:name] if options[:image].is_a?(Hash) - options[:services]&.map! do |service| - if service.is_a?(Hash) - service[:name] - else - service - end - end - end - end - - expose :timeout do |model| - model.timeout - end - - expose :variables - expose :depends_on_builds, using: Build - - expose :credentials, using: BuildCredentials - end - - class Runner < Grape::Entity - expose :id, :token - end - - class RunnerProject < Grape::Entity - expose :id, :project_id, :runner_id - end - - class WebHook < Grape::Entity - expose :id, :project_id, :url - end - - class TriggerRequest < Grape::Entity - expose :id, :variables - expose :pipeline, using: Commit, as: :commit - end - end - end -end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb deleted file mode 100644 index a40b6ab6c9f..00000000000 --- a/lib/ci/api/helpers.rb +++ /dev/null @@ -1,89 +0,0 @@ -module Ci - module API - module Helpers - BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze - BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 10 * 60 - - def authenticate_runners! - forbidden! unless runner_registration_token_valid? - end - - def authenticate_runner! - forbidden! unless current_runner - end - - def authenticate_build! - build = Ci::Build.find_by_id(params[:id]) - - validate_build!(build) do - forbidden! unless build_token_valid?(build) - end - - build - end - - def validate_build!(build) - not_found! unless build - - yield if block_given? - - project = build.project - forbidden!('Project has been deleted!') if project.nil? || project.pending_delete? - forbidden!('Build has been erased!') if build.erased? - end - - def runner_registration_token_valid? - ActiveSupport::SecurityUtils.variable_size_secure_compare( - params[:token], - current_application_settings.runners_registration_token) - end - - def build_token_valid?(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - - # We require to also check `runners_token` to maintain compatibility with old version of runners - token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) - end - - def update_runner_info - return unless update_runner? - - current_runner.contacted_at = Time.now - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - - def update_runner? - # Use a random threshold to prevent beating DB updates. - # It generates a distribution between [40m, 80m]. - # - contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - - current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= contacted_at_max_age - end - - def build_not_found! - if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? / - no_content! - else - not_found! - end - end - - def current_runner - @runner ||= Runner.find_by_token(params[:token].to_s) - end - - def get_runner_version_from_params - return unless params["info"].present? - attributes_for_keys(%w(name version revision platform architecture), params["info"]) - end - - def max_artifacts_size - current_application_settings.max_artifacts_size.megabytes.to_i - end - end - end -end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb deleted file mode 100644 index 45aa2adccf5..00000000000 --- a/lib/ci/api/runners.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Ci - module API - class Runners < Grape::API - resource :runners do - desc 'Delete a runner' - params do - requires :token, type: String, desc: 'The unique token of the runner' - end - delete "delete" do - authenticate_runner! - - status(200) - Ci::Runner.find_by_token(params[:token]).destroy - end - - desc 'Register a new runner' do - success Entities::Runner - end - params do - requires :token, type: String, desc: 'The unique token of the runner' - optional :description, type: String, desc: 'The description of the runner' - optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for' - optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs' - optional :locked, type: Boolean, desc: 'Lock this runner for this specific project' - end - post "register" do - runner_params = declared(params, include_missing: false).except(:token) - - runner = - if runner_registration_token_valid? - # Create shared runner. Requires admin access - Ci::Runner.create(runner_params.merge(is_shared: true)) - elsif project = Project.find_by(runners_token: params[:token]) - # Create a specific runner for project. - project.runners.create(runner_params) - end - - return forbidden! unless runner - - if runner.id - runner.update(get_runner_version_from_params) - present runner, with: Entities::Runner - else - not_found! - end - end - end - end - end -end diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb deleted file mode 100644 index 6225203f223..00000000000 --- a/lib/ci/api/triggers.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Ci - module API - class Triggers < Grape::API - resource :projects do - desc 'Trigger a GitLab CI project build' do - success Entities::TriggerRequest - end - params do - requires :id, type: Integer, desc: 'The ID of a CI project' - requires :ref, type: String, desc: "The name of project's branch or tag" - requires :token, type: String, desc: 'The unique token of the trigger' - optional :variables, type: Hash, desc: 'Optional build variables' - end - post ":id/refs/:ref/trigger" do - project = Project.find_by(ci_id: params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token]) - not_found! unless project && trigger - unauthorized! unless trigger.project == project - - # Validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - # create request and trigger builds - result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref], variables) - pipeline = result.pipeline - - if pipeline.persisted? - present result.trigger_request, with: Entities::TriggerRequest - else - render_validation_error!(pipeline) - end - end - end - end - end -end -- cgit v1.2.1 From 34a859ee2aace209a4331f277ff03d50de3e64be Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 18 Aug 2017 13:02:30 +0100 Subject: Fix MR diffs background migration when file modes are missing Guess the modes based on the following: 1. If the file didn't exist, it's zero. 2. If the diff contains 'Subproject commit', it might be a submodule, so 0600. 3. Otherwise, it's 0644. This isn't perfect, but it doesn't have to be - it won't change file modes in the repository. --- .../deserialize_merge_request_diffs_and_commits.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 310a69a4bd4..3fde1b09efb 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -81,12 +81,15 @@ module Gitlab relative_order: index ) - # Compatibility with old diffs created with Psych. diff_hash.tap do |hash| diff_text = hash[:diff] hash[:too_large] = !!hash[:too_large] + hash[:a_mode] ||= guess_mode(hash[:new_file], hash[:diff]) + hash[:b_mode] ||= guess_mode(hash[:deleted_file], hash[:diff]) + + # Compatibility with old diffs created with Psych. if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only? hash[:binary] = true hash[:diff] = [diff_text].pack('m0') @@ -97,6 +100,15 @@ module Gitlab [commit_rows, file_rows] end + # This doesn't have to be 100% accurate, because it's only used for + # display - it won't change file modes in the repository. Submodules are + # created as 600, regular files as 644. + def guess_mode(file_missing, diff) + return '0' if file_missing + + diff.include?('Subproject commit') ? '160000' : '100644' + end + # Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as # valid, because we don't render them usefully anyway. def valid_raw_diffs?(diffs) -- cgit v1.2.1 From 66afd1ee1cfdefbcc971ab438081e85b6dde044b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 17 Aug 2017 18:34:26 +0200 Subject: Add Gitaly annotation --- lib/gitlab/git/repository.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 53aa5b12489..246bfd57787 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -126,6 +126,8 @@ module Gitlab # This is to work around a bug in libgit2 that causes in-memory refs to # be stale/invalid when packed-refs is changed. # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/474 def find_branch(name, force_reload = false) reload_rugged if force_reload -- cgit v1.2.1 From faaed0aa1f6b26f6b8f69f1c59ae7faa8ccf9ce5 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 17 Aug 2017 18:34:43 +0200 Subject: Delete unused Gitlab::Git methods --- lib/gitlab/git/repository.rb | 20 -------------------- 1 file changed, 20 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 246bfd57787..5152d212a69 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -64,7 +64,6 @@ module Gitlab end delegate :empty?, - :bare?, to: :rugged delegate :exists?, to: :gitaly_repository_client @@ -233,10 +232,6 @@ module Gitlab branch_names + tag_names end - def has_commits? - !empty? - end - # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -594,11 +589,6 @@ module Gitlab raise InvalidRef.new("Invalid reference #{start_point}") end - # Return an array of this repository's remote names - def remote_names - rugged.remotes.each_name.to_a - end - # Delete the specified remote from this repository. def remote_delete(remote_name) rugged.remotes.delete(remote_name) @@ -618,16 +608,6 @@ module Gitlab rugged.remotes.set_url(remote_name, options[:url]) if options[:url] end - # Fetch the specified remote - def fetch(remote_name) - rugged.remotes[remote_name].fetch - end - - # Push +*refspecs+ to the remote identified by +remote_name+. - def push(remote_name, *refspecs) - rugged.remotes[remote_name].push(refspecs) - end - AUTOCRLF_VALUES = { "true" => true, "false" => false, -- cgit v1.2.1 From a902166475772521201196f3f9984c663c6c2ee5 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 18 Aug 2017 15:55:30 +0200 Subject: More annotations --- lib/gitlab/git/repository.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 5152d212a69..163ffb6efc5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -571,6 +571,8 @@ module Gitlab end # Delete the specified branch from the repository + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def delete_branch(branch_name) rugged.branches.delete(branch_name) end @@ -580,6 +582,8 @@ module Gitlab # Examples: # create_branch("feature") # create_branch("other-feature", "master") + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) -- cgit v1.2.1 From df7f2b135a1d811761272ef0f9140ae5aa16c01d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 18 Aug 2017 15:59:17 +0200 Subject: Simplify method arguments --- lib/gitlab/git/repository.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 163ffb6efc5..39263d9e6fc 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -607,9 +607,9 @@ module Gitlab # # Example # repo.update_remote("origin", url: "path/to/repo") - def remote_update(remote_name, options = {}) + def remote_update(remote_name, url:) # TODO: Implement other remote options - rugged.remotes.set_url(remote_name, options[:url]) if options[:url] + rugged.remotes.set_url(remote_name, url) end AUTOCRLF_VALUES = { -- cgit v1.2.1 From 3049dfaf48676ecdf53d0d07297432fbaf4ca720 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 18 Aug 2017 16:03:27 +0200 Subject: Simplify return values --- lib/gitlab/git/repository.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 39263d9e6fc..eb3731ba35a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -596,11 +596,13 @@ module Gitlab # Delete the specified remote from this repository. def remote_delete(remote_name) rugged.remotes.delete(remote_name) + nil end - # Add a new remote to this repository. Returns a Rugged::Remote object + # Add a new remote to this repository. def remote_add(remote_name, url) rugged.remotes.create(remote_name, url) + nil end # Update the specified remote using the values in the +options+ hash @@ -610,6 +612,7 @@ module Gitlab def remote_update(remote_name, url:) # TODO: Implement other remote options rugged.remotes.set_url(remote_name, url) + nil end AUTOCRLF_VALUES = { -- cgit v1.2.1 From 5b37f21bf2c3a9bc6006bcdfbcbc04d19b9b9b86 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Fri, 18 Aug 2017 15:06:41 -0400 Subject: Reduce duplication in GitAccess spec around error messages - Adds a new `ProjectMovedError` class to encapsulate that error condition. Inherits from `NotFoundError` so existing rescues should continue to work. - Separating that condition out of `NotFoundError` allowed us to simplify the `raise_not_found` helper and avoid repeating the literal string. - Spec makes use of `ERROR_MESSAGES` hash to avoid repeating literal error message strings. --- lib/gitlab/git_access.rb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 0b62911958d..3e8b83c0f90 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -4,6 +4,7 @@ module Gitlab class GitAccess UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) + ProjectMovedError = Class.new(NotFoundError) ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', @@ -90,18 +91,18 @@ module Gitlab end def check_project_moved! - if redirected_path - url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - message = <<-MESSAGE.strip_heredoc - Project '#{redirected_path}' was moved to '#{project.full_path}'. + return unless redirected_path - Please update your Git remote and try again: + url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + message = <<-MESSAGE.strip_heredoc + Project '#{redirected_path}' was moved to '#{project.full_path}'. - git remote set-url origin #{url} - MESSAGE + Please update your Git remote and try again: - raise NotFoundError, message - end + git remote set-url origin #{url} + MESSAGE + + raise ProjectMovedError, message end def check_command_disabled!(cmd) -- cgit v1.2.1 From 718ecd4eb5976d1005664dfbd84ee9d9c263e3b9 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 20 Aug 2017 03:28:25 -0700 Subject: Fix Error 500s when attempting to destroy a protected tag Due to a missing `on_delete: :cascade`, users would hit the error that looked like: ``` PG::ForeignKeyViolation: ERROR: update or delete on table "protected_tags" violates foreign key constraint "fk_rails_f7dfda8c51" on table "protected_tag_create_access_levels" DETAIL: Key (id)=(1385) is still referenced from table "protected_tag_create_access_levels". : DELETE FROM "protected_tags" WHERE "protected_tags"."id" = 1385 ``` Closes #36013 --- lib/gitlab/database/migration_helpers.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 69ca9aa596b..b83e633c7ed 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -606,6 +606,11 @@ module Gitlab Arel::Nodes::SqlLiteral.new(replace.to_sql) end end + + def remove_foreign_key_without_error(*args) + remove_foreign_key(*args) + rescue ArgumentError + end end end end -- cgit v1.2.1 From 0db5f576fedfa5c4b2a1f9f01a0fdc4cbcd759f9 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Mon, 21 Aug 2017 12:50:09 +0100 Subject: Only require sidekiq-limit_fetch when enabled in settings This gem allows Sidekiq jobs to be throttled. Unfortunately, it has a side-effect: when we haven't enabled job throttling, it will still hit Redis a lot (and miss, because nothing is configured). As this setting already required a restart, ensure that the library is only required when it's enabled. --- lib/gitlab/sidekiq_throttler.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb index d4d39a888e7..5512afa45a8 100644 --- a/lib/gitlab/sidekiq_throttler.rb +++ b/lib/gitlab/sidekiq_throttler.rb @@ -3,6 +3,8 @@ module Gitlab class << self def execute! if Gitlab::CurrentSettings.sidekiq_throttling_enabled? + require 'sidekiq-limit_fetch' + Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| Sidekiq::Queue[queue].limit = queue_limit end -- cgit v1.2.1 From fb9bd552c95883ed142111c9988f57fd2b531876 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 21 Aug 2017 16:35:31 +0200 Subject: Fix fork MRs importing issue --- lib/gitlab/import_export/import_export.yml | 2 ++ lib/gitlab/import_export/merge_request_parser.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9d9ebcb389a..9e08a5a8fb9 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -133,5 +133,7 @@ methods: - :utf8_diff merge_requests: - :diff_head_sha + - :source_branch_sha + - :target_branch_sha project: - :description_html diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index c20adc20bfd..81a213e8321 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -30,7 +30,7 @@ module Gitlab end def branch_exists?(branch_name) - @project.repository.branch_exists?(branch_name) + @project.repository.raw.branch_exists?(branch_name) end def fork_merge_request? -- cgit v1.2.1 From b7ebb447d7a0f8e65c791d4f7af6e042b72ff644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Mon, 21 Aug 2017 15:13:40 -0300 Subject: Correctly encode string params for Gitaly's TreeEntries RPC --- lib/gitlab/gitaly_client/commit_service.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index b36e81278d6..2d58fb0186e 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -80,8 +80,8 @@ module Gitlab def tree_entries(repository, revision, path) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, - revision: revision, - path: path.presence || '.' + revision: GitalyClient.encode(revision), + path: path.present? ? GitalyClient.encode(path) : '.' ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) -- cgit v1.2.1 From 9e6fa996eab978506af1084b79a9c3f91f6d575b Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 4 Aug 2017 07:30:42 +0200 Subject: New storage is now "Hashed" instead of "UUID" --- lib/gitlab/import_export/import_export.yml | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9d9ebcb389a..894950e341f 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -98,6 +98,7 @@ excluded_attributes: - :last_activity_at - :last_repository_updated_at - :last_repository_check_at + - :storage_version snippets: - :expired_at merge_request_diff: -- cgit v1.2.1 From 78823675b24e82e73a523ad98f1dec78bec6976c Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Fri, 11 Aug 2017 12:03:35 +0200 Subject: Prevent using gitlab import task when hashed storage is enabled --- lib/tasks/gitlab/import.rake | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 48bd9139ce8..6e10ba374bf 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -11,6 +11,12 @@ namespace :gitlab do # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + if Project.current_application_settings.hashed_storage_enabled + puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red) + + exit 1 + end + Gitlab.config.repositories.storages.each_value do |repository_storage| git_base_path = repository_storage['path'] repos_to_import = Dir.glob(git_base_path + '/**/*.git') -- cgit v1.2.1 From d17a7be8308b06b7077a7cffc5d258148ee08c87 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 16 Aug 2017 04:49:54 +0200 Subject: Refactor project and storage types --- lib/backup/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 88821ae56e0..4e92be85110 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -75,7 +75,7 @@ module Backup path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - project.ensure_storage_path_exist + project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) -- cgit v1.2.1 From fe9fc0af7519a2809116d73a8322d881e146e3f8 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 22 Aug 2017 15:12:27 +0900 Subject: Enfouce namespace for Kubernetes to lowercase --- lib/gitlab/regex.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 1adc5ec952a..58f6245579a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -53,7 +53,8 @@ module Gitlab end def kubernetes_namespace_regex_message - "can contain only letters, digits or '-', and cannot start or end with '-'" + "can contain only lowercase letters, digits, and '-'. " \ + "Must start with a letter, and cannot end with '-'" end def environment_slug_regex -- cgit v1.2.1 From 6e8d0b78ebbde2eada151649fc7d1040b902e28f Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 15 Aug 2017 13:56:04 +0100 Subject: Use event-based waiting in Gitlab::JobWaiter --- lib/gitlab/job_waiter.rb | 57 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 208f0e1bbea..4d6bbda15f3 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -1,12 +1,31 @@ module Gitlab # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + # + # Its use requires the cooperation of the sidekiq jobs themselves. Set up the + # waiter, then start the jobs, passing them its `key`. Their `perform` methods + # should look like: + # + # def perform(args, notify_key) + # # do work + # ensure + # ::Gitlab::JobWaiter.notify(notify_key, jid) + # end + # + # The JobWaiter blocks popping items from a Redis array. All the sidekiq jobs + # push to that array when done. Once the waiter has popped `count` items, it + # knows all the jobs are done. class JobWaiter - # The sleep interval between checking keys, in seconds. - INTERVAL = 0.1 + def self.notify(key, jid) + Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } + end + + attr_reader :key, :jobs_remaining, :finished - # jobs - The job IDs to wait for. - def initialize(jobs) - @jobs = jobs + # jobs_remaining - the number of jobs left to wait for + def initialize(jobs_remaining) + @key = "gitlab:job_waiter:#{SecureRandom.uuid}" + @jobs_remaining = jobs_remaining + @finished = [] end # Waits for all the jobs to be completed. @@ -15,13 +34,33 @@ module Gitlab # ensures we don't indefinitely block a caller in case a job takes # long to process, or is never processed. def wait(timeout = 10) - start = Time.current + deadline = Time.now.utc + timeout + + Gitlab::Redis::SharedState.with do |redis| + # Fallback key expiry: allow a long grace period to reduce the chance of + # a job pushing to an expired key and recreating it + redis.expire(key, [timeout * 2, 10.minutes.to_i].max) + + while jobs_remaining > 0 + # Redis will not take fractional seconds. Prefer waiting too long over + # not waiting long enough + seconds_left = (deadline - Time.now.utc).ceil - while (Time.current - start) <= timeout - break if SidekiqStatus.all_completed?(@jobs) + # Redis interprets 0 as "wait forever", so skip the final `blpop` call + break if seconds_left <= 0 - sleep(INTERVAL) # to not overload Redis too much. + list, jid = redis.blpop(key, timeout: seconds_left) + break unless list && jid # timed out + + @finished << jid + @jobs_remaining -= 1 + end + + # All jobs have finished, so expire the key immediately + redis.expire(key, 0) if jobs_remaining == 0 end + + finished end end end -- cgit v1.2.1 From 193b199672f5229dd6d2cff30fc8c794bb774bbd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 Aug 2017 12:47:20 +0200 Subject: Add Sidekiq migration helpers for migrating queues --- lib/gitlab/database/migration_helpers.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index b83e633c7ed..ecb23e79e80 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -611,6 +611,20 @@ module Gitlab remove_foreign_key(*args) rescue ArgumentError end + + def sidekiq_queue_migrate(queue_from, to: queue_to) + while sidekiq_queue_length(queue_from) > 0 + Sidekiq.redis do |conn| + conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" + end + end + end + + def sidekiq_queue_length(queue_name) + Sidekiq.redis do |conn| + conn.llen("queue:#{queue_name}") + end + end end end end -- cgit v1.2.1 From b0f09406f50882c7e085c2a9b853be5dcf3d79dd Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Tue, 22 Aug 2017 14:04:54 +0100 Subject: Always return a simple diff viewer We didn't have a fallback case before, because we believed the conditions were exhaustive. They weren't, so we can always fallback to not previewing. --- lib/gitlab/diff/file.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 6d7de52cb80..17a9ec01637 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -250,6 +250,8 @@ module Gitlab DiffViewer::Renamed elsif mode_changed? DiffViewer::ModeChanged + else + DiffViewer::NoPreview end end -- cgit v1.2.1 From 258d5a50e63d5e29b6a3adc0a250727a8232695b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Wed, 9 Aug 2017 17:47:11 -0400 Subject: Incorporate DiffService.CommitPatch Gitaly RPC --- lib/gitlab/git/commit.rb | 8 +++++++- lib/gitlab/gitaly_client/commit_service.rb | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index a499bbc6266..5ee6669050c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -271,7 +271,13 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 def to_diff - rugged_diff_from_parent.patch + Gitlab::GitalyClient.migrate(:commit_patch) do |is_enabled| + if is_enabled + @repository.gitaly_commit_client.patch(id) + else + rugged_diff_from_parent.patch + end + end end # Returns a diff object for the changes from this commit's first parent. diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index b36e81278d6..2a984b97e92 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -194,6 +194,16 @@ module Gitlab response.commit end + def patch(revision) + request = Gitaly::CommitPatchRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request) + + response.sum(&:data) + end + private def commit_diff_request_params(commit, options = {}) -- cgit v1.2.1 From 4598e0c3924b30d11495e803e88a6ded11094318 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 22 Aug 2017 17:09:45 -0400 Subject: Fix a potential timeout in `Gitlab::Logger.read_latest` If this method was called for a file that didn't exist, we attempted to first `build` it. But if the file wasn't writeable, such as a symlink pointing to an unmounted filesystem, the method would just hang and eventually timeout. Further, this was entirely pointless since we were creating a file and then shelling out to `tail`, eventually returning an empty Array. Now we just skip building it and return the empty Array straight away. --- lib/gitlab/logger.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 59b21149a9a..6bffd410ed0 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -14,13 +14,9 @@ module Gitlab def self.read_latest path = Rails.root.join("log", file_name) - self.build unless File.exist?(path) - tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path})) - tail_output.split("\n") - end - def self.read_latest_for(filename) - path = Rails.root.join("log", filename) + return [] unless File.readable?(path) + tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path})) tail_output.split("\n") end -- cgit v1.2.1 From faf92651aa5e9c6bcb88acac8de27c878d7edf06 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 23 Aug 2017 08:58:55 +0200 Subject: Fix invalid default argument in migration helpers --- lib/gitlab/database/migration_helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index ecb23e79e80..5e2c6cc5cad 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -612,7 +612,7 @@ module Gitlab rescue ArgumentError end - def sidekiq_queue_migrate(queue_from, to: queue_to) + def sidekiq_queue_migrate(queue_from, to:) while sidekiq_queue_length(queue_from) > 0 Sidekiq.redis do |conn| conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" -- cgit v1.2.1 From 9b9309329207449ef022bdcf06bff5f8eae36032 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 22 Aug 2017 13:05:36 +0200 Subject: Decouple GitOperationService from User --- lib/gitlab/git/committer.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 lib/gitlab/git/committer.rb (limited to 'lib') diff --git a/lib/gitlab/git/committer.rb b/lib/gitlab/git/committer.rb new file mode 100644 index 00000000000..ef3576eaa1c --- /dev/null +++ b/lib/gitlab/git/committer.rb @@ -0,0 +1,21 @@ +module Gitlab + module Git + class Committer + attr_reader :name, :email, :gl_id + + def self.from_user(user) + new(user.name, user.email, Gitlab::GlId.gl_id(user)) + end + + def initialize(name, email, gl_id) + @name = name + @email = email + @gl_id = gl_id + end + + def ==(other) + [name, email, gl_id] == [other.name, other.email, other.gl_id] + end + end + end +end -- cgit v1.2.1 From 65f83941c39c14c2af9da5064393545ea2f7b3e5 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 22 Aug 2017 13:54:14 +0200 Subject: Make gl_repository a G::G::Repository attribute --- lib/gitlab/git/hook.rb | 13 ++++++++----- lib/gitlab/git/repository.rb | 5 +++-- 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 8f0c377ef4f..08cede42ba2 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -6,15 +6,18 @@ module Gitlab module Git class Hook GL_PROTOCOL = 'web'.freeze - attr_reader :name, :repo_path, :path + attr_reader :name, :path, :repository - def initialize(name, project) + def initialize(name, repository) @name = name - @project = project - @repo_path = project.repository.path + @repository = repository @path = File.join(repo_path.strip, 'hooks', name) end + def repo_path + repository.path + end + def exists? File.exist?(path) end @@ -44,7 +47,7 @@ module Gitlab 'GL_ID' => gl_id, 'PWD' => repo_path, 'GL_PROTOCOL' => GL_PROTOCOL, - 'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false) + 'GL_REPOSITORY' => repository.gl_repository } options = { diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index eb3731ba35a..976ea7e4035 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -49,13 +49,14 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage + attr_reader :storage, :gl_repository, :relative_path # 'path' must be the path to a _bare_ git repository, e.g. # /path/to/my-repo.git - def initialize(storage, relative_path) + def initialize(storage, relative_path, gl_repository) @storage = storage @relative_path = relative_path + @gl_repository = gl_repository storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) -- cgit v1.2.1 From dc7c6bede27abd4507072276ef23b40a74ee297a Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 22 Aug 2017 14:18:09 +0200 Subject: Move GitHooksService to Gitlab::Git --- lib/gitlab/git/hook.rb | 4 ++-- lib/gitlab/git/hooks_service.rb | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 lib/gitlab/git/hooks_service.rb (limited to 'lib') diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 08cede42ba2..cc35d77c6e4 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,6 +1,6 @@ -# Gitaly note: JV: looks like this is only used by GitHooksService in +# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in # app/services. We shouldn't bother migrating this until we know how -# GitHooksService will be migrated. +# Gitlab::Git::HooksService will be migrated. module Gitlab module Git diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb new file mode 100644 index 00000000000..1da92fcc0e2 --- /dev/null +++ b/lib/gitlab/git/hooks_service.rb @@ -0,0 +1,37 @@ +module Gitlab + module Git + class HooksService + PreReceiveError = Class.new(StandardError) + + attr_accessor :oldrev, :newrev, :ref + + def execute(committer, repository, oldrev, newrev, ref) + @repository = repository + @gl_id = committer.gl_id + @oldrev = oldrev + @newrev = newrev + @ref = ref + + %w(pre-receive update).each do |hook_name| + status, message = run_hook(hook_name) + + unless status + raise PreReceiveError, message + end + end + + yield(self).tap do + run_hook('post-receive') + end + end + + private + + def run_hook(name) + hook = Gitlab::Git::Hook.new(name, @repository) + hook.trigger(@gl_id, oldrev, newrev, ref) + end + end + end +end + -- cgit v1.2.1 From da769135fede9dececee8ab3e1f6951de7361e7f Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 22 Aug 2017 16:27:15 +0200 Subject: Rubocop whitespace --- lib/gitlab/git/committer.rb | 2 +- lib/gitlab/git/hooks_service.rb | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/committer.rb b/lib/gitlab/git/committer.rb index ef3576eaa1c..1f4bcf7a3a0 100644 --- a/lib/gitlab/git/committer.rb +++ b/lib/gitlab/git/committer.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Committer attr_reader :name, :email, :gl_id - + def self.from_user(user) new(user.name, user.email, Gitlab::GlId.gl_id(user)) end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb index 1da92fcc0e2..ea8a87a1290 100644 --- a/lib/gitlab/git/hooks_service.rb +++ b/lib/gitlab/git/hooks_service.rb @@ -2,31 +2,31 @@ module Gitlab module Git class HooksService PreReceiveError = Class.new(StandardError) - + attr_accessor :oldrev, :newrev, :ref - + def execute(committer, repository, oldrev, newrev, ref) @repository = repository @gl_id = committer.gl_id @oldrev = oldrev @newrev = newrev @ref = ref - + %w(pre-receive update).each do |hook_name| status, message = run_hook(hook_name) - + unless status raise PreReceiveError, message end end - + yield(self).tap do run_hook('post-receive') end end - + private - + def run_hook(name) hook = Gitlab::Git::Hook.new(name, @repository) hook.trigger(@gl_id, oldrev, newrev, ref) @@ -34,4 +34,3 @@ module Gitlab end end end - -- cgit v1.2.1 From d8d2b73b9f17e5af9aeccb1e9ba40045486651b5 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 18 Aug 2017 17:21:01 +0200 Subject: Improve bare repository import - Allow imports into nested groups - Make sure it sets the correct visibility level when creating new groups While doing this, I moved the import into a testable class, that made it easier to improve. --- lib/gitlab/bare_repository_importer.rb | 127 +++++++++++++++++++++++++++++++++ lib/tasks/gitlab/import.rake | 65 +---------------- 2 files changed, 129 insertions(+), 63 deletions(-) create mode 100644 lib/gitlab/bare_repository_importer.rb (limited to 'lib') diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb new file mode 100644 index 00000000000..4bc3ccc3a1a --- /dev/null +++ b/lib/gitlab/bare_repository_importer.rb @@ -0,0 +1,127 @@ +module Gitlab + class BareRepositoryImporter + NoAdminError = Class.new(StandardError) + + def self.execute + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + git_base_path = repository_storage['path'] + repos_to_import = Dir.glob(git_base_path + '/**/*.git') + + repos_to_import.each do |repo_path| + if repo_path.end_with?('.wiki.git') + log " * Skipping wiki repo" + next + end + + log "Processing #{repo_path}".color(:yellow) + + repo_relative_path = repo_path[repository_storage['path'].length..-1] + .sub(/^\//, '') # Remove leading `/` + .sub(/\.git$/, '') # Remove `.git` at the end + new(storage_name, repo_relative_path).create_project_if_needed + end + end + end + + attr_reader :storage_name, :full_path, :group_path, :project_path, :user + delegate :log, to: :class + + def initialize(storage_name, repo_path) + @storage_name = storage_name + @full_path = repo_path + + unless @user = User.admins.order_id_asc.first + raise NoAdminError.new('No admin user found to import repositories') + end + + @group_path, @project_path = File.split(repo_path) + @group_path = nil if @group_path == '.' + end + + def create_project_if_needed + if project = Project.find_by_full_path(full_path) + log " * #{project.name} (#{full_path}) exists" + return project + end + + create_project + end + + private + + def create_project + group = find_or_create_group + + project_params = { + name: project_path, + path: project_path, + repository_storage: storage_name, + namespace_id: group&.id + } + + project = Projects::CreateService.new(user, project_params).execute + + if project.persisted? + log " * Created #{project.name} (#{full_path})".color(:green) + ProjectCacheWorker.perform_async(project.id) + else + log " * Failed trying to create #{project.name} (#{full_path})".color(:red) + log " Errors: #{project.errors.messages}".color(:red) + end + + project + end + + def find_or_create_group + return nil unless group_path + + if namespace = Namespace.find_by_full_path(group_path) + log " * Namespace #{group_path} exists.".color(:green) + return namespace + end + + create_group_path + end + + def create_group_path + group_path_segments = group_path.split('/') + + new_group, parent_group = nil + partial_path_segments = [] + while subgroup_name = group_path_segments.shift + partial_path_segments << subgroup_name + partial_path = partial_path_segments.join('/') + + unless new_group = Group.find_by_full_path(partial_path) + log " * Creating group #{partial_path}.".color(:green) + params = { + path: subgroup_name, + name: subgroup_name, + parent: parent_group, + visibility_level: Gitlab::CurrentSettings.current_application_settings.default_group_visibility + } + new_group = Groups::CreateService.new(user, params).execute + end + + if new_group.persisted? + log " * Group #{partial_path} (#{new_group.id}) available".color(:green) + else + log " * Failed trying to create group #{partial_path}.".color(:red) + log " * Errors: #{new_group.errors.messages}".color(:red) + end + parent_group = new_group + end + + new_group + end + + # This is called from within a rake task only used by Admins, so allow writing + # to STDOUT + # + # rubocop:disable Rails/Output + def self.log(message) + puts message + end + # rubocop:enable Rails/Output + end +end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 6e10ba374bf..d227a0c8bdb 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -9,6 +9,7 @@ namespace :gitlab do # * The project owner will set to the first administator of the system # * Existing projects will be skipped # + # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do if Project.current_application_settings.hashed_storage_enabled @@ -17,69 +18,7 @@ namespace :gitlab do exit 1 end - Gitlab.config.repositories.storages.each_value do |repository_storage| - git_base_path = repository_storage['path'] - repos_to_import = Dir.glob(git_base_path + '/**/*.git') - - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' - - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' - - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end - - project = Project.find_by_full_path(path) - - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first - - project_params = { - name: name, - path: name - } - - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(name: group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) - end - end - # set project group - project_params[:namespace_id] = group.id - end - - project = Projects::CreateService.new(user, project_params).execute - - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - ProjectCacheWorker.perform_async(project.id) - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) - end - end - end - end - - puts "Done!".color(:green) + Gitlab::BareRepositoryImporter.execute end end end -- cgit v1.2.1 From 22ef4ba3a4be66296e5ee9231b4eb39e172c0f1f Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 22 Aug 2017 12:13:25 +0200 Subject: Migrate creation of nested groups into a service --- lib/gitlab/bare_repository_importer.rb | 35 ++-------------------------------- lib/tasks/import.rake | 18 +---------------- 2 files changed, 3 insertions(+), 50 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb index 4bc3ccc3a1a..9323bfc7fb2 100644 --- a/lib/gitlab/bare_repository_importer.rb +++ b/lib/gitlab/bare_repository_importer.rb @@ -80,39 +80,8 @@ module Gitlab return namespace end - create_group_path - end - - def create_group_path - group_path_segments = group_path.split('/') - - new_group, parent_group = nil - partial_path_segments = [] - while subgroup_name = group_path_segments.shift - partial_path_segments << subgroup_name - partial_path = partial_path_segments.join('/') - - unless new_group = Group.find_by_full_path(partial_path) - log " * Creating group #{partial_path}.".color(:green) - params = { - path: subgroup_name, - name: subgroup_name, - parent: parent_group, - visibility_level: Gitlab::CurrentSettings.current_application_settings.default_group_visibility - } - new_group = Groups::CreateService.new(user, params).execute - end - - if new_group.persisted? - log " * Group #{partial_path} (#{new_group.id}) available".color(:green) - else - log " * Failed trying to create group #{partial_path}.".color(:red) - log " * Errors: #{new_group.errors.messages}".color(:red) - end - parent_group = new_group - end - - new_group + log " * Creating Group: #{group_path}" + Groups::NestedCreateService.new(user, group_path: group_path).execute end # This is called from within a rake task only used by Admins, so allow writing diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 96b8f59242c..1206302cb76 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -72,23 +72,7 @@ class GithubImport return @current_user.namespace if names == @current_user.namespace_path return @current_user.namespace unless @current_user.can_create_group? - full_path_namespace = Namespace.find_by_full_path(names) - - return full_path_namespace if full_path_namespace - - names.split('/').inject(nil) do |parent, name| - begin - namespace = Group.create!(name: name, - path: name, - owner: @current_user, - parent: parent) - namespace.add_owner(@current_user) - - namespace - rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid - Namespace.where(parent: parent).find_by_path_or_name(name) - end - end + Groups::NestedCreateService.new(@current_user, group_path: names).execute end def full_path_namespace(names) -- cgit v1.2.1 From f76d8c902a08040eeaacc8c7a3a9506fb55501e7 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 22 Aug 2017 14:03:40 +0200 Subject: Fix error when importing a GitHub-wiki repository This would occur when Wiki's are enabled on GitHub, but there is no wiki repository --- lib/github/import.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/github/import.rb b/lib/github/import.rb index 4cc01593ef4..7b848081e85 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -107,7 +107,7 @@ module Github # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. if e.message !~ /repository not exported/ - errors(:wiki, wiki_url, e.message) + error(:wiki, wiki_url, e.message) end end -- cgit v1.2.1 From 9240d3552a0850dc3a7a661012c51af12c9f6457 Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Wed, 23 Aug 2017 13:44:54 +0200 Subject: Properly encode Gitaly RawBlame request params --- lib/gitlab/gitaly_client/commit_service.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 8fb7341b2dc..57f42bd35ee 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -175,8 +175,8 @@ module Gitlab def raw_blame(revision, path) request = Gitaly::RawBlameRequest.new( repository: @gitaly_repo, - revision: revision, - path: path + revision: GitalyClient.encode(revision), + path: GitalyClient.encode(path) ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request) -- cgit v1.2.1 From 502d6464b07154d74eecbeddbf2cd6dba841380f Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Wed, 23 Aug 2017 13:01:11 +0100 Subject: Allow v4 API GET requests for groups to be unauthenticated --- lib/api/groups.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 49c3b2278c7..892fd239df4 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -2,7 +2,7 @@ module API class Groups < Grape::API include PaginationParams - before { authenticate! } + before { authenticate_non_get! } helpers do params :optional_params_ce do @@ -48,10 +48,10 @@ module API end get do groups = if params[:owned] - current_user.owned_groups - elsif current_user.admin + current_user ? current_user.owned_groups : Group.none + elsif current_user&.admin? Group.all - elsif params[:all_available] + elsif params[:all_available] || current_user.nil? GroupsFinder.new(current_user).execute else current_user.groups -- cgit v1.2.1 From c1cf5f41018dd4cf0523c6a80c8617651d88658c Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 23 Aug 2017 19:18:13 +0200 Subject: Support simple string LDAP attribute specifications, and search for name rather than username attributes --- lib/gitlab/ldap/adapter.rb | 6 +----- lib/gitlab/ldap/person.rb | 9 +++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 8867a91c244..cd7e4ca7b7e 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -73,7 +73,7 @@ module Gitlab private def user_options(field, value, limit) - options = { attributes: user_attributes } + options = { attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq } options[:size] = limit if limit if field.to_sym == :dn @@ -99,10 +99,6 @@ module Gitlab filter end end - - def user_attributes - %W(#{config.uid} cn dn) + config.attributes['username'] + config.attributes['email'] - end end end end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index e138b466a34..4d6f8ac79de 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -21,6 +21,15 @@ module Gitlab adapter.dn_matches_filter?(dn, AD_USER_DISABLED) end + def self.ldap_attributes(config) + [ + 'dn', # Used in `dn` + config.uid, # Used in `uid` + *config.attributes['name'], # Used in `name` + *config.attributes['email'] # Used in `email` + ] + end + def initialize(entry, provider) Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry -- cgit v1.2.1 From 37904108b965eecabdbe631ca2f3465a3cf18a1e Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 23 Aug 2017 22:06:42 -0700 Subject: Fix inconsistent number of branches when remote branches are present Users of project mirrors would see that the number of branches did not match the number in the branch dropdown because remote branches were counted when Rugged was in use. With Gitaly, only local branches are counted. Closes #36934 --- lib/gitlab/git/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index eb3731ba35a..f5747951d5e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -153,7 +153,7 @@ module Gitlab if is_enabled gitaly_ref_client.count_branch_names else - rugged.branches.count do |ref| + rugged.branches.each(:local).count do |ref| begin ref.name && ref.target # ensures the branch is valid -- cgit v1.2.1 From fb49c94e49149a2043b774ba33daa3fe79febdd4 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Thu, 24 Aug 2017 09:20:04 +0000 Subject: Delegate Repository::branch_exists? and ref_exists? to Gitlab::Git --- lib/gitlab/git/repository.rb | 23 +++++++++++++++++++++++ lib/gitlab/gitaly_client/ref_service.rb | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f5747951d5e..860ed01c05d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -201,6 +201,19 @@ module Gitlab end end + # Returns true if the given ref name exists + # + # Ref names must start with `refs/`. + def ref_exists?(ref_name) + gitaly_migrate(:ref_exists) do |is_enabled| + if is_enabled + gitaly_ref_exists?(ref_name) + else + rugged_ref_exists?(ref_name) + end + end + end + # Returns true if the given tag exists # # name - The name of the tag as a String. @@ -989,6 +1002,16 @@ module Gitlab raw_output.compact end + # Returns true if the given ref name exists + # + # Ref names must start with `refs/`. + def rugged_ref_exists?(ref_name) + raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/') + rugged.references.exist?(ref_name) + rescue Rugged::ReferenceError + false + end + # Returns true if the given ref name exists # # Ref names must start with `refs/`. diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index cdcfed36740..8c0008c6971 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -71,7 +71,7 @@ module Gitlab end def ref_exists?(ref_name) - request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: ref_name) + request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: GitalyClient.encode(ref_name)) response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) response.value rescue GRPC::InvalidArgument => e -- cgit v1.2.1 From e8525e107da9234d743caf8a0c7db1f46af60e89 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Tue, 22 Aug 2017 16:27:09 +0100 Subject: Show un-highlighted diffs when blobs are the same For some old merge requests, we don't have enough information to figure out the old blob and the new blob for the file. This means that we can't highlight the diff correctly, but we can still display it without highlighting. --- lib/gitlab/diff/file.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 17a9ec01637..1dabd4ebdd0 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -186,7 +186,10 @@ module Gitlab end def content_changed? - old_blob && new_blob && old_blob.id != new_blob.id + return blobs_changed? if diff_refs + return false if new_file? || deleted_file? || renamed_file? + + text? && diff_lines.any? end def different_type? @@ -225,6 +228,10 @@ module Gitlab private + def blobs_changed? + old_blob && new_blob && old_blob.id != new_blob.id + end + def simple_viewer_class return DiffViewer::NotDiffable unless diffable? -- cgit v1.2.1 From 5ced2d8d7d9107f031894c5b16908db8bf6b913f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 23 Aug 2017 15:09:20 +0200 Subject: Rename CI/CD job triggering policy class to Policy --- lib/gitlab/ci/config/entry/job.rb | 4 ++-- lib/gitlab/ci/config/entry/policy.rb | 18 ++++++++++++++++++ lib/gitlab/ci/config/entry/trigger.rb | 18 ------------------ 3 files changed, 20 insertions(+), 20 deletions(-) create mode 100644 lib/gitlab/ci/config/entry/policy.rb delete mode 100644 lib/gitlab/ci/config/entry/trigger.rb (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 32f5c6ab142..91aac6df4b1 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -59,10 +59,10 @@ module Gitlab entry :services, Entry::Services, description: 'Services that will be used to execute this job.' - entry :only, Entry::Trigger, + entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.' - entry :except, Entry::Trigger, + entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' entry :variables, Entry::Variables, diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb new file mode 100644 index 00000000000..a08ab8a9d14 --- /dev/null +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a trigger policy for the job. + # + class Policy < Node + include Validatable + + validations do + validates :config, array_of_strings_or_regexps: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb deleted file mode 100644 index 16b234e6c59..00000000000 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents a trigger policy for the job. - # - class Trigger < Node - include Validatable - - validations do - validates :config, array_of_strings_or_regexps: true - end - end - end - end - end -end -- cgit v1.2.1 From 2adff699cea2cf1e60180d7eae73dfe5e8a09235 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 24 Aug 2017 11:33:06 +0100 Subject: Refactor complicated API group finding rules into GroupsFinder --- lib/api/groups.rb | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 892fd239df4..e56427304a6 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -47,16 +47,8 @@ module API use :pagination end get do - groups = if params[:owned] - current_user ? current_user.owned_groups : Group.none - elsif current_user&.admin? - Group.all - elsif params[:all_available] || current_user.nil? - GroupsFinder.new(current_user).execute - else - current_user.groups - end - + find_params = { all_available: params[:all_available], owned: params[:owned] } + groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) -- cgit v1.2.1 From 0d7d7c1057c80e930d56363f2efd0519e1462586 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 24 Aug 2017 14:54:27 +0200 Subject: Use aspect-oriented design in CI/CD config entries --- lib/gitlab/ci/config/entry/configurable.rb | 3 ++- lib/gitlab/ci/config/entry/node.rb | 11 ++++++----- lib/gitlab/ci/config/entry/validatable.rb | 11 +++++++++++ lib/gitlab/ci/config/entry/validator.rb | 2 +- 4 files changed, 20 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index e05aca9881b..68b6742385a 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -15,9 +15,10 @@ module Gitlab # module Configurable extend ActiveSupport::Concern - include Validatable included do + include Validatable + validations do validates :config, type: Hash end diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index a6a914d79c1..2474684e07f 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -16,8 +16,9 @@ module Gitlab @metadata = metadata @entries = {} - @validator = self.class.validator.new(self) - @validator.validate(:new) + self.class.aspects.to_a.each do |aspect| + instance_exec(&aspect) + end end def [](key) @@ -47,7 +48,7 @@ module Gitlab end def errors - @validator.messages + descendants.flat_map(&:errors) + [] end def value @@ -79,8 +80,8 @@ module Gitlab def self.default end - def self.validator - Validator + def self.aspects + @aspects ||= [] end end end diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb index f7f1b111571..5ced778d311 100644 --- a/lib/gitlab/ci/config/entry/validatable.rb +++ b/lib/gitlab/ci/config/entry/validatable.rb @@ -5,6 +5,17 @@ module Gitlab module Validatable extend ActiveSupport::Concern + def self.included(node) + node.aspects.append -> do + @validator = self.class.validator.new(self) + @validator.validate(:new) + end + end + + def errors + @validator.messages + descendants.flat_map(&:errors) + end + class_methods do def validator @validator ||= Class.new(Entry::Validator).tap do |validator| diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 55343005fe3..5ab54d7e218 100644 --- a/lib/gitlab/ci/config/entry/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -30,7 +30,7 @@ module Gitlab def key_name if key.blank? - @entry.class.name.demodulize.underscore.humanize + @entry.class.name.to_s.demodulize.underscore.humanize else key end -- cgit v1.2.1 From 8c409fc40ba4bf2e7fe0c8458fd2b59c09bd123a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 25 Aug 2017 09:49:18 +0200 Subject: Make it possible to define CI/CD config strategies --- lib/gitlab/ci/config/entry/policy.rb | 25 ++++++++++++++++++++---- lib/gitlab/ci/config/entry/simplifiable.rb | 31 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 lib/gitlab/ci/config/entry/simplifiable.rb (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index a08ab8a9d14..ac564a6c7d0 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -5,11 +5,28 @@ module Gitlab ## # Entry that represents a trigger policy for the job. # - class Policy < Node - include Validatable + class Policy < Simplifiable + strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } + strategy :ExpressionsPolicy, if: -> (config) { config.is_a?(Hash) } - validations do - validates :config, array_of_strings_or_regexps: true + class RefsPolicy < Entry::Node + include Entry::Validatable + + validations do + validates :config, array_of_strings_or_regexps: true + end + + def value + { refs: @config } + end + end + + class ExpressionsPolicy < Entry::Node + include Entry::Validatable + + validations do + validates :config, type: Hash + end end end end diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb new file mode 100644 index 00000000000..5319065b846 --- /dev/null +++ b/lib/gitlab/ci/config/entry/simplifiable.rb @@ -0,0 +1,31 @@ +module Gitlab + module Ci + class Config + module Entry + class Simplifiable < SimpleDelegator + EntryStrategy = Struct.new(:name, :condition) + + def initialize(config, **metadata) + strategy = self.class.strategies.find do |variant| + variant.condition.call(config) + end + + entry = self.class.const_get(strategy.name) + + super(entry.new(config, metadata)) + end + + def self.strategy(name, **opts) + EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| + (@strategies ||= []).append(strategy) + end + end + + def self.strategies + @strategies || [] + end + end + end + end + end +end -- cgit v1.2.1 From fcb4d1f80945cbcdfbdb73e102d046447f55e5b5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 25 Aug 2017 10:27:00 +0200 Subject: Implement complex only/except policy CI/CD config --- lib/gitlab/ci/config/entry/policy.rb | 20 ++++++++++++++++++-- lib/gitlab/ci/config/entry/simplifiable.rb | 12 ++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index ac564a6c7d0..decacebee55 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -3,7 +3,7 @@ module Gitlab class Config module Entry ## - # Entry that represents a trigger policy for the job. + # Entry that represents an only/except trigger policy for the job. # class Policy < Simplifiable strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } @@ -23,9 +23,25 @@ module Gitlab class ExpressionsPolicy < Entry::Node include Entry::Validatable + include Entry::Attributable + + attributes :refs, :expressions validations do - validates :config, type: Hash + validates :config, presence: true + validates :config, allowed_keys: %i[refs expressions] + + with_options allow_nil: true do + validates :refs, array_of_strings_or_regexps: true + validates :expressions, type: Array + validates :expressions, presence: true + end + end + end + + class UnknownStrategy < Entry::Node + def errors + ['policy has to be either an array of conditions or a hash'] end end end diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb index 5319065b846..c26576fcbcd 100644 --- a/lib/gitlab/ci/config/entry/simplifiable.rb +++ b/lib/gitlab/ci/config/entry/simplifiable.rb @@ -10,7 +10,7 @@ module Gitlab variant.condition.call(config) end - entry = self.class.const_get(strategy.name) + entry = self.class.entry_class(strategy) super(entry.new(config, metadata)) end @@ -22,7 +22,15 @@ module Gitlab end def self.strategies - @strategies || [] + @strategies.to_a + end + + def self.entry_class(strategy) + if strategy.present? + self.const_get(strategy.name) + else + self::UnknownStrategy + end end end end -- cgit v1.2.1 From fa2915ec2deaf9aa73ae18a60836a831376da943 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 24 Aug 2017 14:46:08 +0100 Subject: Fix searching for files by path --- lib/gitlab/file_finder.rb | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 093d9ed8092..10ffc345bd5 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -6,27 +6,48 @@ module Gitlab attr_reader :project, :ref + delegate :repository, to: :project + def initialize(project, ref) @project = project @ref = ref end def find(query) - blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE) - found_file_names = Set.new + by_content = find_by_content(query) - results = blobs.map do |blob| - blob = Gitlab::ProjectSearchResults.parse_search_result(blob) - found_file_names << blob.filename + already_found = Set.new(by_content.map(&:filename)) + by_filename = find_by_filename(query, except: already_found) - [blob.filename, blob] - end + (by_content + by_filename) + .sort_by(&:filename) + .map { |blob| [blob.filename, blob] } + end - project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename| - results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename) - end + private - results.sort_by(&:first) + def find_by_content(query) + results = repository.search_files_by_content(query, ref).first(BATCH_SIZE) + results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result) } + end + + def find_by_filename(query, except: []) + filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) + filenames.delete_if { |filename| except.include?(filename) } unless except.empty? + + blob_refs = filenames.map { |filename| [ref, filename] } + blobs = Gitlab::Git::Blob.batch(repository, blob_refs, blob_size_limit: 1024) + + blobs.map do |blob| + Gitlab::SearchResults::FoundBlob.new( + id: blob.id, + filename: blob.path, + basename: File.basename(blob.path), + ref: ref, + startline: 1, + data: blob.data + ) + end end end end -- cgit v1.2.1 From 9e203582b367a1b84035572261a79b62e22bfeaa Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 23 Aug 2017 01:51:53 +0900 Subject: Improve AutocompleteController#user.json performance --- lib/gitlab/sql/pattern.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 lib/gitlab/sql/pattern.rb (limited to 'lib') diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb new file mode 100644 index 00000000000..47ea19994a2 --- /dev/null +++ b/lib/gitlab/sql/pattern.rb @@ -0,0 +1,29 @@ +module Gitlab + module SQL + class Pattern + MIN_CHARS_FOR_PARTIAL_MATCHING = 3 + + attr_reader :query + + def initialize(query) + @query = query + end + + def to_sql + if exact_matching? + query + else + "%#{query}%" + end + end + + def exact_matching? + !partial_matching? + end + + def partial_matching? + @query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING + end + end + end +end -- cgit v1.2.1 From a061a2461a05200ab534d382b67e1b9099128478 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 25 Aug 2017 11:42:40 +0200 Subject: Fix CI/CD trigger policy default value --- lib/gitlab/ci/config/entry/policy.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index decacebee55..41a3737b34a 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -44,6 +44,9 @@ module Gitlab ['policy has to be either an array of conditions or a hash'] end end + + def self.default + end end end end -- cgit v1.2.1 From 946e8d3a93eb8c9e30d1f3baa3b5b28e6c06fbc1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 25 Aug 2017 12:01:59 +0200 Subject: Use only/except policy that returns an array --- lib/gitlab/ci/config/entry/policy.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 41a3737b34a..495822e9f5a 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -15,10 +15,6 @@ module Gitlab validations do validates :config, array_of_strings_or_regexps: true end - - def value - { refs: @config } - end end class ExpressionsPolicy < Entry::Node -- cgit v1.2.1 From 99dddac55850fed68ea6f66363c3f083bd4866fa Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 25 Aug 2017 13:00:45 +0200 Subject: Simplify ci config entry validator implementation --- lib/gitlab/ci/config/entry/validator.rb | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 5ab54d7e218..83bca0d08bc 100644 --- a/lib/gitlab/ci/config/entry/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -24,16 +24,11 @@ module Gitlab private def location - predecessors = ancestors.map(&:key).compact - predecessors.append(key_name).join(':') + ancestors.map(&:key).compact.append(key_name).join(':') end def key_name - if key.blank? - @entry.class.name.to_s.demodulize.underscore.humanize - else - key - end + key.presence || @entry.class.name.to_s.demodulize.underscore.humanize end end end -- cgit v1.2.1 From 7e6bc4dde29af635c4ec281beea20ca87ccfbe34 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 25 Aug 2017 15:16:09 +0200 Subject: Improve reporting of a CI/CD entry config location --- lib/gitlab/ci/config/entry/attributable.rb | 2 ++ lib/gitlab/ci/config/entry/node.rb | 7 +++++++ lib/gitlab/ci/config/entry/policy.rb | 2 +- lib/gitlab/ci/config/entry/validator.rb | 11 ----------- 4 files changed, 10 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 1c8b55ee4c4..24ff862a142 100644 --- a/lib/gitlab/ci/config/entry/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -8,6 +8,8 @@ module Gitlab class_methods do def attributes(*attributes) attributes.flatten.each do |attribute| + raise ArgumentError if method_defined?(attribute) + define_method(attribute) do return unless config.is_a?(Hash) diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index 2474684e07f..c868943c42e 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -71,6 +71,13 @@ module Gitlab true end + def location + name = @key.presence || self.class.name.to_s.demodulize + .underscore.humanize.downcase + + ancestors.map(&:key).append(name).compact.join(':') + end + def inspect val = leaf? ? config : descendants unspecified = specified? ? '' : '(unspecified) ' diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 495822e9f5a..05602f1b3c6 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -37,7 +37,7 @@ module Gitlab class UnknownStrategy < Entry::Node def errors - ['policy has to be either an array of conditions or a hash'] + ["#{location} has to be either an array of conditions or a hash"] end end diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 83bca0d08bc..2df23a3edcd 100644 --- a/lib/gitlab/ci/config/entry/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -8,7 +8,6 @@ module Gitlab def initialize(entry) super(entry) - @entry = entry end def messages @@ -20,16 +19,6 @@ module Gitlab def self.name 'Validator' end - - private - - def location - ancestors.map(&:key).compact.append(key_name).join(':') - end - - def key_name - key.presence || @entry.class.name.to_s.demodulize.underscore.humanize - end end end end -- cgit v1.2.1 From d1054bd3ddb48c15b6a3a53f8c57974208094106 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Fri, 25 Aug 2017 22:38:07 +0800 Subject: Resolve feedback on the MR: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/13766 * Rename AfterImportService * Use constants * Move Git operations to Gitlab::Git::Repository * Use Regexp.union --- lib/gitlab/git/repository.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index eb3731ba35a..f6d30ad7534 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -232,6 +232,13 @@ module Gitlab branch_names + tag_names end + # Returns an Array of all ref names, except when it's matching pattern + # + # regexp - The pattern for ref names we don't want + def all_ref_names_except(regexp) + rugged.references.reject { |ref| ref.name =~ regexp }.map(&:name) + end + # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -577,6 +584,10 @@ module Gitlab rugged.branches.delete(branch_name) end + def delete_refs(ref_names) + ref_names.each(&rugged.references.method(:delete)) + end + # Create a new branch named **ref+ based on **stat_point+, HEAD by default # # Examples: -- cgit v1.2.1 From 04108611716c0f1bb00092077ad5e31785675490 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 25 Aug 2017 18:26:55 +0200 Subject: Add specs for attributable aspect of ci config entry --- lib/gitlab/ci/config/entry/attributable.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 24ff862a142..3e87a09704e 100644 --- a/lib/gitlab/ci/config/entry/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -8,7 +8,9 @@ module Gitlab class_methods do def attributes(*attributes) attributes.flatten.each do |attribute| - raise ArgumentError if method_defined?(attribute) + if method_defined?(attribute) + raise ArgumentError, 'Method already defined!' + end define_method(attribute) do return unless config.is_a?(Hash) -- cgit v1.2.1 From 5ce9e03f6c015ab28aac1b3490acbf6f1c6f538f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 26 Aug 2017 11:11:28 +0200 Subject: Use new complex only/except policy internal scheme --- lib/ci/gitlab_ci_yaml_processor.rb | 2 +- lib/gitlab/ci/config/entry/policy.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3a4911b23b0..3efd9b3bdac 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -22,7 +22,7 @@ module Ci def jobs_for_ref(ref, tag = false, source = nil) @jobs.select do |_, job| - process?(job[:only], job[:except], ref, tag, source) + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) end end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 05602f1b3c6..bcb76de65b2 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -15,6 +15,10 @@ module Gitlab validations do validates :config, array_of_strings_or_regexps: true end + + def value + { refs: @config } + end end class ExpressionsPolicy < Entry::Node -- cgit v1.2.1 From 326dc7da3bb7e6537095277dc8ee8ae880774b62 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 26 Aug 2017 12:45:36 +0200 Subject: Check if kubernetes required before creating a job --- lib/ci/gitlab_ci_yaml_processor.rb | 19 +++++++++++++++++-- lib/gitlab/ci/config/entry/policy.rb | 11 +++++------ 2 files changed, 22 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3efd9b3bdac..72a38e97648 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -44,6 +44,22 @@ module Ci end end + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_available? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes & !except_kubernetes, + only_kubernetes & has_kubernetes, + except_kubernetes & !has_kubernetes].any? + end + end + def builds @jobs.map do |name, _| build_attributes(name) @@ -52,8 +68,7 @@ module Ci def stage_seeds(pipeline) seeds = @stages.uniq.map do |stage| - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) + builds = pipeline_stage_builds(stage, pipeline) Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index bcb76de65b2..a8bba3d3ea4 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -7,7 +7,7 @@ module Gitlab # class Policy < Simplifiable strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } - strategy :ExpressionsPolicy, if: -> (config) { config.is_a?(Hash) } + strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } class RefsPolicy < Entry::Node include Entry::Validatable @@ -21,20 +21,19 @@ module Gitlab end end - class ExpressionsPolicy < Entry::Node + class ComplexPolicy < Entry::Node include Entry::Validatable include Entry::Attributable - attributes :refs, :expressions + attributes :refs, :kubernetes validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs expressions] + validates :config, allowed_keys: %i[refs kubernetes] with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true - validates :expressions, type: Array - validates :expressions, presence: true + validates :kubernetes, inclusion: { in: [true] } end end end -- cgit v1.2.1 From 866aab7f2a92f9929a5c5811d3d3c23c11184b26 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sat, 26 Aug 2017 22:32:55 +0900 Subject: Fix escape characters was not sanitized --- lib/gitlab/sql/pattern.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 47ea19994a2..46c973d8a11 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -11,9 +11,9 @@ module Gitlab def to_sql if exact_matching? - query + sanitized_query else - "%#{query}%" + "%#{sanitized_query}%" end end @@ -24,6 +24,11 @@ module Gitlab def partial_matching? @query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end + + def sanitized_query + # Note: ActiveRecord::Base.sanitize_sql_like is a protected method + ActiveRecord::Base.__send__(:sanitize_sql_like, query) + end end end end -- cgit v1.2.1 From 1e53f40c25a87a285ce6f35b5ae1717fe87477ae Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 27 Aug 2017 14:43:42 -0700 Subject: Simplify system hook testing and guarantee test will fire The change in !11728 would cause an arbitrary project to be chosen to test system hooks, and it's likely that the project would not have any commits or relevant commits to test the hook. This would prevent admins from verifying that the hook fired. Instead of trying to create a representative hook dynamically, just send static data to guarantee the hook will actually be tested. Closes #37067 --- lib/gitlab/data_builder/push.rb | 33 +++++++++++++++++++++++++++++++++ lib/gitlab/data_builder/repository.rb | 21 +++++++++++++++++++++ 2 files changed, 54 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 5c5f507d44d..4ab5b3455a5 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -3,6 +3,35 @@ module Gitlab module Push extend self + SAMPLE_DATA = + { + object_kind: "push", + event_name: "push", + before: "95790bf891e76fee5e1747ab589903a6a1f80f22", + after: "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + ref: "refs/heads/master", + checkout_sha: "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + message: "Hello World", + user_id: 4, + user_name: "John Smith", + user_email: "john@example.com", + user_avatar: "https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80", + project_id: 15, + commits: [ + { + id: "c5feabde2d8cd023215af4d2ceeb7a64839fc428", + message: "Add simple search to projects in public area", + timestamp: "2013-05-13T18:18:08+00:00", + url: "https://test.example.com/gitlab/gitlabhq/commit/c5feabde2d8cd023215af4d2ceeb7a64839fc428", + author: { + name: "Test User", + email: "test@example.com" + } + } + ], + total_commits_count: 1 + }.freeze + # Produce a hash of post-receive data # # data = { @@ -74,6 +103,10 @@ module Gitlab build(project, user, commits.last&.id, commits.first&.id, ref, commits) end + def sample_data + SAMPLE_DATA + end + private def checkout_sha(repository, newrev, ref) diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb index b42dc052949..c9c13ec6487 100644 --- a/lib/gitlab/data_builder/repository.rb +++ b/lib/gitlab/data_builder/repository.rb @@ -3,6 +3,23 @@ module Gitlab module Repository extend self + SAMPLE_DATA = { + event_name: 'repository_update', + user_id: 10, + user_name: 'john.doe', + user_email: 'test@example.com', + user_avatar: 'http://example.com/avatar/user.png', + project_id: 40, + changes: [ + { + before: "8205ea8d81ce0c6b90fbe8280d118cc9fdad6130", + after: "4045ea7a3df38697b3730a20fb73c8bed8a3e69e", + ref: "refs/heads/master" + } + ], + "refs": ["refs/heads/master"] + }.freeze + # Produce a hash of post-receive data def update(project, user, changes, refs) { @@ -30,6 +47,10 @@ module Gitlab ref: ref } end + + def sample_data + SAMPLE_DATA + end end end end -- cgit v1.2.1 From eef5135e0826b5f43a9229c82d482215546c7cf5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Mon, 28 Aug 2017 10:46:18 +0200 Subject: Fix events error importing GitLab projects --- lib/gitlab/import_export/import_export.yml | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index e5d4bb686e7..78795dd3d92 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -138,3 +138,7 @@ methods: - :target_branch_sha project: - :description_html + events: + - :action + push_event_payload: + - :action \ No newline at end of file -- cgit v1.2.1 From 998afa5f74558be215a924d95aa131a69831ca43 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Wed, 1 Mar 2017 14:35:48 +0100 Subject: API: Respect the 'If-Unmodified-Since' for delete endpoints --- lib/api/access_requests.rb | 3 +++ lib/api/award_emoji.rb | 1 + lib/api/boards.rb | 1 + lib/api/broadcast_messages.rb | 1 + lib/api/deploy_keys.rb | 2 ++ lib/api/environments.rb | 1 + lib/api/groups.rb | 2 ++ lib/api/helpers.rb | 8 ++++++++ lib/api/issues.rb | 2 ++ lib/api/labels.rb | 2 ++ lib/api/members.rb | 3 ++- lib/api/merge_requests.rb | 2 ++ lib/api/notes.rb | 2 ++ lib/api/project_hooks.rb | 2 ++ lib/api/project_snippets.rb | 2 ++ lib/api/projects.rb | 2 ++ lib/api/runners.rb | 2 ++ lib/api/services.rb | 2 ++ lib/api/snippets.rb | 1 + lib/api/system_hooks.rb | 2 ++ lib/api/triggers.rb | 2 ++ lib/api/users.rb | 28 ++++++++++++++++++++++++++++ 22 files changed, 72 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index cdacf9839e5..0c5b8862d79 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -67,6 +67,9 @@ module API end delete ":id/access_requests/:user_id" do source = find_source(source_type, params[:id]) + member = source.public_send(:requesters).find_by!(user_id: params[:user_id]) + + check_unmodified_since(member.updated_at) status 204 ::Members::DestroyService.new(source, current_user, params) diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 5a028fc9d0b..51a8587d26e 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -85,6 +85,7 @@ module API end delete "#{endpoint}/:award_id" do award = awardable.award_emoji.find(params[:award_id]) + check_unmodified_since(award.updated_at) unauthorized! unless award.user == current_user || current_user.admin? diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 5a2d7a681e3..d36df77dc6c 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -124,6 +124,7 @@ module API authorize!(:admin_list, user_project) list = board_lists.find(params[:list_id]) + check_unmodified_since(list.updated_at) service = ::Boards::Lists::DestroyService.new(user_project, current_user) diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 9980aec4752..352972b584a 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -90,6 +90,7 @@ module API end delete ':id' do message = find_message + check_unmodified_since(message.updated_at) status 204 message.destroy diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 42e7c1486b0..971cc816454 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -125,6 +125,8 @@ module API key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) not_found!('Deploy Key') unless key + check_unmodified_since(key.updated_at) + status 204 key.destroy end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index c774a5c6685..3fc423ae79a 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -78,6 +78,7 @@ module API authorize! :update_environment, user_project environment = user_project.environments.find(params[:environment_id]) + check_unmodified_since(environment.updated_at) status 204 environment.destroy diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e56427304a6..c9b32a85487 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -117,6 +117,8 @@ module API delete ":id" do group = find_group!(params[:id]) authorize! :admin_group, group + + check_unmodified_since(group.updated_at) status 204 ::Groups::DestroyService.new(group, current_user).execute diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index b56fd2388b3..1c74a14d91c 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -11,6 +11,14 @@ module API declared(params, options).to_h.symbolize_keys end + def check_unmodified_since(last_modified) + if_unmodified_since = Time.parse(headers['If-Unmodified-Since']) if headers.key?('If-Unmodified-Since') rescue nil + + if if_unmodified_since && if_unmodified_since < last_modified + render_api_error!('412 Precondition Failed', 412) + end + end + def current_user return @current_user if defined?(@current_user) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4cec1145f3a..cee9898d3a6 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -230,6 +230,8 @@ module API not_found!('Issue') unless issue authorize!(:destroy_issue, issue) + check_unmodified_since(issue.updated_at) + status 204 issue.destroy end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 4520c98d951..45fa57fdf55 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -56,6 +56,8 @@ module API label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label + check_unmodified_since(label.updated_at) + status 204 label.destroy end diff --git a/lib/api/members.rb b/lib/api/members.rb index bb970b7cd54..5634f123eca 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -94,7 +94,8 @@ module API delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) # Ensure that memeber exists - source.members.find_by!(user_id: params[:user_id]) + member = source.members.find_by!(user_id: params[:user_id]) + check_unmodified_since(member.updated_at) status 204 ::Members::DestroyService.new(source, current_user, declared_params).execute diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 8810d4e441d..c6fecc1aa6c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -164,6 +164,8 @@ module API merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) + check_unmodified_since(merge_request.updated_at) + status 204 merge_request.destroy end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 4e4e473994b..58d71787aca 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -129,7 +129,9 @@ module API end delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note + check_unmodified_since(note.updated_at) status 204 ::Notes::DestroyService.new(user_project, current_user).execute(note) diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 649dd891f56..74d736fda59 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -96,6 +96,8 @@ module API delete ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) + check_unmodified_since(hook.updated_at) + status 204 hook.destroy end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index f3d905b0068..645162d564d 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -116,6 +116,8 @@ module API not_found!('Snippet') unless snippet authorize! :admin_project_snippet, snippet + check_unmodified_since(snippet.updated_at) + status 204 snippet.destroy end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 15c3832b032..eab0ca0b3c9 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -334,6 +334,8 @@ module API desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project + check_unmodified_since(user_project.updated_at) + ::Projects::DestroyService.new(user_project, current_user, {}).async_execute accepted! diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 31f940fe96b..e3b2eb904b7 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -77,7 +77,9 @@ module API end delete ':id' do runner = get_runner(params[:id]) + authenticate_delete_runner!(runner) + check_unmodified_since(runner.updated_at) status 204 runner.destroy! diff --git a/lib/api/services.rb b/lib/api/services.rb index 843c05ae32e..4fef3383e5e 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -655,6 +655,8 @@ module API end delete ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) + # Todo, not sure + check_unmodified_since(service.updated_at) attrs = service_attributes(service).inject({}) do |hash, key| hash.merge!(key => nil) diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 35ece56c65c..7107b3d669c 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -122,6 +122,7 @@ module API return not_found!('Snippet') unless snippet authorize! :destroy_personal_snippet, snippet + check_unmodified_since(snippet.updated_at) status 204 snippet.destroy diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index c0179037440..64066a75b15 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -66,6 +66,8 @@ module API hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook + check_unmodified_since(hook.updated_at) + status 204 hook.destroy end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index edfdb63d183..4ae70c65759 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -140,6 +140,8 @@ module API trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger + check_unmodified_since(trigger.updated_at) + status 204 trigger.destroy end diff --git a/lib/api/users.rb b/lib/api/users.rb index e2019d6d512..942bb72cf97 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -230,7 +230,12 @@ module API key = user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key +<<<<<<< HEAD status 204 +======= + check_unmodified_since(key.updated_at) + +>>>>>>> API: Respect the 'If-Unmodified-Since' for delete endpoints key.destroy end @@ -287,7 +292,14 @@ module API email = user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email +<<<<<<< HEAD Emails::DestroyService.new(user, email: email.email).execute +======= + check_unmodified_since(email.updated_at) + + email.destroy + user.update_secondary_emails! +>>>>>>> API: Respect the 'If-Unmodified-Since' for delete endpoints end desc 'Delete a user. Available only for admins.' do @@ -299,11 +311,18 @@ module API end delete ":id" do authenticated_as_admin! + user = User.find_by(id: params[:id]) not_found!('User') unless user +<<<<<<< HEAD status 204 user.delete_async(deleted_by: current_user, params: params) +======= + check_unmodified_since(user.updated_at) + + ::Users::DestroyService.new(current_user).execute(user) +>>>>>>> API: Respect the 'If-Unmodified-Since' for delete endpoints end desc 'Block a user. Available only for admins.' @@ -481,6 +500,8 @@ module API key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key + check_unmodified_since(key.updated_at) + status 204 key.destroy end @@ -533,6 +554,7 @@ module API email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email +<<<<<<< HEAD status 204 Emails::DestroyService.new(current_user, email: email.email).execute end @@ -550,6 +572,12 @@ module API .reorder(last_activity_on: :asc) present paginate(activities), with: Entities::UserActivity +======= + check_unmodified_since(email.updated_at) + + email.destroy + current_user.update_secondary_emails! +>>>>>>> API: Respect the 'If-Unmodified-Since' for delete endpoints end end end -- cgit v1.2.1 From e80313f9ee5b3495a8713e6ddae111bc8106155b Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Thu, 2 Mar 2017 13:14:13 +0100 Subject: Conditionally destroy a ressource --- lib/api/access_requests.rb | 11 +++++----- lib/api/award_emoji.rb | 4 +--- lib/api/boards.rb | 11 +++++----- lib/api/broadcast_messages.rb | 4 +--- lib/api/deploy_keys.rb | 5 +---- lib/api/environments.rb | 4 +--- lib/api/groups.rb | 7 +++---- lib/api/helpers.rb | 17 +++++++++++++--- lib/api/issues.rb | 4 +--- lib/api/labels.rb | 5 +---- lib/api/members.rb | 7 +++---- lib/api/merge_requests.rb | 4 +--- lib/api/notes.rb | 6 +++--- lib/api/project_hooks.rb | 5 +---- lib/api/project_snippets.rb | 4 +--- lib/api/projects.rb | 5 +++-- lib/api/runner.rb | 6 ++++-- lib/api/runners.rb | 7 ++----- lib/api/services.rb | 4 ++-- lib/api/snippets.rb | 4 +--- lib/api/system_hooks.rb | 5 +---- lib/api/triggers.rb | 5 +---- lib/api/users.rb | 47 ++++++++++++------------------------------- 23 files changed, 69 insertions(+), 112 deletions(-) (limited to 'lib') diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 0c5b8862d79..4fa9b2b2494 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -67,13 +67,12 @@ module API end delete ":id/access_requests/:user_id" do source = find_source(source_type, params[:id]) - member = source.public_send(:requesters).find_by!(user_id: params[:user_id]) + member = source.requesters.find_by!(user_id: params[:user_id]) - check_unmodified_since(member.updated_at) - - status 204 - ::Members::DestroyService.new(source, current_user, params) - .execute(:requesters) + destroy_conditionally!(member) do + ::Members::DestroyService.new(source, current_user, params) + .execute(:requesters) + end end end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 51a8587d26e..8e3851640da 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -85,12 +85,10 @@ module API end delete "#{endpoint}/:award_id" do award = awardable.award_emoji.find(params[:award_id]) - check_unmodified_since(award.updated_at) unauthorized! unless award.user == current_user || current_user.admin? - status 204 - award.destroy + destroy_conditionally!(award) end end end diff --git a/lib/api/boards.rb b/lib/api/boards.rb index d36df77dc6c..0d11c5fc971 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -122,14 +122,13 @@ module API end delete "/lists/:list_id" do authorize!(:admin_list, user_project) - list = board_lists.find(params[:list_id]) - check_unmodified_since(list.updated_at) - - service = ::Boards::Lists::DestroyService.new(user_project, current_user) - unless service.execute(list) - render_api_error!({ error: 'List could not be deleted!' }, 400) + destroy_conditionally!(list) do |list| + service = ::Boards::Lists::DestroyService.new(user_project, current_user) + unless service.execute(list) + render_api_error!({ error: 'List could not be deleted!' }, 400) + end end end end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 352972b584a..0b45621ce7b 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -90,10 +90,8 @@ module API end delete ':id' do message = find_message - check_unmodified_since(message.updated_at) - status 204 - message.destroy + destroy_conditionally!(message) end end end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 971cc816454..f405c341398 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -125,10 +125,7 @@ module API key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) not_found!('Deploy Key') unless key - check_unmodified_since(key.updated_at) - - status 204 - key.destroy + destroy_conditionally!(key) end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 3fc423ae79a..e33269f9483 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -78,10 +78,8 @@ module API authorize! :update_environment, user_project environment = user_project.environments.find(params[:environment_id]) - check_unmodified_since(environment.updated_at) - status 204 - environment.destroy + destroy_conditionally!(environment) end desc 'Stops an existing environment' do diff --git a/lib/api/groups.rb b/lib/api/groups.rb index c9b32a85487..ee2ad27837b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -117,11 +117,10 @@ module API delete ":id" do group = find_group!(params[:id]) authorize! :admin_group, group - - check_unmodified_since(group.updated_at) - status 204 - ::Groups::DestroyService.new(group, current_user).execute + destroy_conditionally!(group) do |group| + ::Groups::DestroyService.new(group, current_user).execute + end end desc 'Get a list of projects in this group.' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 1c74a14d91c..8d4f8c01903 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -11,14 +11,25 @@ module API declared(params, options).to_h.symbolize_keys end - def check_unmodified_since(last_modified) - if_unmodified_since = Time.parse(headers['If-Unmodified-Since']) if headers.key?('If-Unmodified-Since') rescue nil + def check_unmodified_since!(last_modified) + if_unmodified_since = Time.parse(headers['If-Unmodified-Since']) rescue nil - if if_unmodified_since && if_unmodified_since < last_modified + if if_unmodified_since && last_modified > if_unmodified_since render_api_error!('412 Precondition Failed', 412) end end + def destroy_conditionally!(resource, last_update_field: :updated_at) + check_unmodified_since!(resource.public_send(last_update_field)) + + status 204 + if block_given? + yield resource + else + resource.destroy + end + end + def current_user return @current_user if defined?(@current_user) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index cee9898d3a6..6503629e2a2 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -230,10 +230,8 @@ module API not_found!('Issue') unless issue authorize!(:destroy_issue, issue) - check_unmodified_since(issue.updated_at) - status 204 - issue.destroy + destroy_conditionally!(issue) end desc 'List merge requests closing issue' do diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 45fa57fdf55..c0cf618ee8d 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -56,10 +56,7 @@ module API label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label - check_unmodified_since(label.updated_at) - - status 204 - label.destroy + destroy_conditionally!(label) end desc 'Update an existing label. At least one optional parameter is required.' do diff --git a/lib/api/members.rb b/lib/api/members.rb index 5634f123eca..a5d3d7f25a0 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -93,12 +93,11 @@ module API end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - # Ensure that memeber exists member = source.members.find_by!(user_id: params[:user_id]) - check_unmodified_since(member.updated_at) - status 204 - ::Members::DestroyService.new(source, current_user, declared_params).execute + destroy_conditionally!(member) do + ::Members::DestroyService.new(source, current_user, declared_params).execute + end end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index c6fecc1aa6c..969c6064662 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -164,10 +164,8 @@ module API merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) - check_unmodified_since(merge_request.updated_at) - status 204 - merge_request.destroy + destroy_conditionally!(merge_request) end params do diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 58d71787aca..e116448c15b 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -131,10 +131,10 @@ module API note = user_project.notes.find(params[:note_id]) authorize! :admin_note, note - check_unmodified_since(note.updated_at) - status 204 - ::Notes::DestroyService.new(user_project, current_user).execute(note) + destroy_conditionally!(note) do |note| + ::Notes::DestroyService.new(user_project, current_user).execute(note) + end end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 74d736fda59..5b457bbe639 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -96,10 +96,7 @@ module API delete ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - check_unmodified_since(hook.updated_at) - - status 204 - hook.destroy + destroy_conditionally!(hook) end end end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 645162d564d..704e8c6718d 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -116,10 +116,8 @@ module API not_found!('Snippet') unless snippet authorize! :admin_project_snippet, snippet - check_unmodified_since(snippet.updated_at) - status 204 - snippet.destroy + destroy_conditionally!(snippet) end desc 'Get a raw project snippet' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index eab0ca0b3c9..36fe3f243b9 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -334,9 +334,10 @@ module API desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project - check_unmodified_since(user_project.updated_at) - ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + destroy_conditionally!(user_project) do + ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + end accepted! end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 88fc62d33df..daa8ddbe251 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -45,8 +45,10 @@ module API end delete '/' do authenticate_runner! - status 204 - Ci::Runner.find_by_token(params[:token]).destroy + + runner = Ci::Runner.find_by_token(params[:token]) + + destroy_conditionally!(runner) end desc 'Validates authentication credentials' do diff --git a/lib/api/runners.rb b/lib/api/runners.rb index e3b2eb904b7..68c2120cc15 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -79,10 +79,8 @@ module API runner = get_runner(params[:id]) authenticate_delete_runner!(runner) - check_unmodified_since(runner.updated_at) - status 204 - runner.destroy! + destroy_conditionally!(runner) end end @@ -137,8 +135,7 @@ module API runner = runner_project.runner forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1 - status 204 - runner_project.destroy + destroy_conditionally!(runner_project) end end diff --git a/lib/api/services.rb b/lib/api/services.rb index 4fef3383e5e..2b5ef75b6bf 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -655,8 +655,8 @@ module API end delete ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - # Todo, not sure - check_unmodified_since(service.updated_at) + # Todo: Check if this done the right way + check_unmodified_since!(service.updated_at) attrs = service_attributes(service).inject({}) do |hash, key| hash.merge!(key => nil) diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 7107b3d669c..00eb7c60f16 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -122,10 +122,8 @@ module API return not_found!('Snippet') unless snippet authorize! :destroy_personal_snippet, snippet - check_unmodified_since(snippet.updated_at) - status 204 - snippet.destroy + destroy_conditionally!(snippet) end desc 'Get a raw snippet' do diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 64066a75b15..6b6a03e3300 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -66,10 +66,7 @@ module API hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook - check_unmodified_since(hook.updated_at) - - status 204 - hook.destroy + destroy_conditionally!(hook) end end end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 4ae70c65759..c9fee7e5193 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -140,10 +140,7 @@ module API trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger - check_unmodified_since(trigger.updated_at) - - status 204 - trigger.destroy + destroy_conditionally!(trigger) end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 942bb72cf97..d7c7b9ae9c1 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -230,13 +230,7 @@ module API key = user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key -<<<<<<< HEAD - status 204 -======= - check_unmodified_since(key.updated_at) - ->>>>>>> API: Respect the 'If-Unmodified-Since' for delete endpoints - key.destroy + destroy_conditionally!(key) end desc 'Add an email address to a specified user. Available only for admins.' do @@ -292,14 +286,11 @@ module API email = user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email -<<<<<<< HEAD - Emails::DestroyService.new(user, email: email.email).execute -======= - check_unmodified_since(email.updated_at) + destroy_conditionally!(email) do |email| + Emails::DestroyService.new(current_user, email: email.email).execute + end - email.destroy user.update_secondary_emails! ->>>>>>> API: Respect the 'If-Unmodified-Since' for delete endpoints end desc 'Delete a user. Available only for admins.' do @@ -315,14 +306,9 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user -<<<<<<< HEAD - status 204 - user.delete_async(deleted_by: current_user, params: params) -======= - check_unmodified_since(user.updated_at) - - ::Users::DestroyService.new(current_user).execute(user) ->>>>>>> API: Respect the 'If-Unmodified-Since' for delete endpoints + destroy_conditionally!(user) do + user.delete_async(deleted_by: current_user, params: params) + end end desc 'Block a user. Available only for admins.' @@ -500,10 +486,7 @@ module API key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key - check_unmodified_since(key.updated_at) - - status 204 - key.destroy + destroy_conditionally!(key) end desc "Get the currently authenticated user's email addresses" do @@ -554,9 +537,11 @@ module API email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email -<<<<<<< HEAD - status 204 - Emails::DestroyService.new(current_user, email: email.email).execute + destroy_conditionally!(email) do |email| + Emails::DestroyService.new(current_user, email: email.email).execute + end + + current_user.update_secondary_emails! end desc 'Get a list of user activities' @@ -572,12 +557,6 @@ module API .reorder(last_activity_on: :asc) present paginate(activities), with: Entities::UserActivity -======= - check_unmodified_since(email.updated_at) - - email.destroy - current_user.update_secondary_emails! ->>>>>>> API: Respect the 'If-Unmodified-Since' for delete endpoints end end end -- cgit v1.2.1 From f0f3f38576c0691e6d0e751c962382beea998afb Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Thu, 2 Mar 2017 14:21:36 +0100 Subject: Use commit date for branches and tags --- lib/api/branches.rb | 15 +++++++++++---- lib/api/tags.rb | 15 +++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/api/branches.rb b/lib/api/branches.rb index d3dbf941298..b87f7cdbad1 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -125,11 +125,18 @@ module API delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project - result = DeleteBranchService.new(user_project, current_user) - .execute(params[:branch]) + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit = user_project.repository.commit(branch.dereferenced_target) + + destroy_conditionally!(commit, last_update_field: :authored_date) do + result = DeleteBranchService.new(user_project, current_user) + .execute(params[:branch]) - if result[:status] != :success - render_api_error!(result[:message], result[:return_code]) + if result[:status] != :success + render_api_error!(result[:message], result[:return_code]) + end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 1333747cced..81b17935b81 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -65,11 +65,18 @@ module API delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do authorize_push_project - result = ::Tags::DestroyService.new(user_project, current_user) - .execute(params[:tag_name]) + tag = user_project.repository.find_tag(params[:tag_name]) + not_found!('Tag') unless tag + + commit = user_project.repository.commit(tag.dereferenced_target) + + destroy_conditionally!(commit, last_update_field: :authored_date) do + result = ::Tags::DestroyService.new(user_project, current_user) + .execute(params[:tag_name]) - if result[:status] != :success - render_api_error!(result[:message], result[:return_code]) + if result[:status] != :success + render_api_error!(result[:message], result[:return_code]) + end end end -- cgit v1.2.1 From dcd4ea473cab20eee05995ecaca043da98ca5a8d Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Thu, 24 Aug 2017 10:41:54 +0200 Subject: Update remaining endpoints --- lib/api/group_variables.rb | 3 +-- lib/api/helpers.rb | 2 +- lib/api/pipeline_schedules.rb | 3 +-- lib/api/projects.rb | 1 - lib/api/protected_branches.rb | 4 +--- lib/api/services.rb | 14 +++++++------- lib/api/users.rb | 7 +++++-- lib/api/variables.rb | 1 + 8 files changed, 17 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index f64da4ab77b..25152f30998 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -88,8 +88,7 @@ module API variable = user_group.variables.find_by(key: params[:key]) not_found!('GroupVariable') unless variable - status 204 - variable.destroy + destroy_conditionally!(variable) end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8d4f8c01903..84980864151 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -14,7 +14,7 @@ module API def check_unmodified_since!(last_modified) if_unmodified_since = Time.parse(headers['If-Unmodified-Since']) rescue nil - if if_unmodified_since && last_modified > if_unmodified_since + if if_unmodified_since && last_modified && last_modified > if_unmodified_since render_api_error!('412 Precondition Failed', 412) end end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index dbeaf9e17ef..e3123ef4e2d 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -117,8 +117,7 @@ module API not_found!('PipelineSchedule') unless pipeline_schedule authorize! :admin_pipeline_schedule, pipeline_schedule - status :accepted - present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails + destroy_conditionally!(pipeline_schedule) end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 36fe3f243b9..78e25b6da03 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,7 +1,6 @@ require_dependency 'declarative_policy' module API - # Projects API class Projects < Grape::API include PaginationParams diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index dccf4fa27a7..15fcb9e8e27 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -76,9 +76,7 @@ module API delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) - protected_branch.destroy - - status 204 + destroy_conditionally!(protected_branch) end end end diff --git a/lib/api/services.rb b/lib/api/services.rb index 2b5ef75b6bf..ff9ddd44439 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -655,15 +655,15 @@ module API end delete ":id/services/:service_slug" do service = user_project.find_or_initialize_service(params[:service_slug].underscore) - # Todo: Check if this done the right way - check_unmodified_since!(service.updated_at) - attrs = service_attributes(service).inject({}) do |hash, key| - hash.merge!(key => nil) - end + destroy_conditionally!(service) do + attrs = service_attributes(service).inject({}) do |hash, key| + hash.merge!(key => nil) + end - unless service.update_attributes(attrs.merge(active: false)) - render_api_error!('400 Bad Request', 400) + unless service.update_attributes(attrs.merge(active: false)) + render_api_error!('400 Bad Request', 400) + end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index d7c7b9ae9c1..96f47bb618a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -408,8 +408,11 @@ module API requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' end delete ':impersonation_token_id' do - status 204 - find_impersonation_token.revoke! + token = find_impersonation_token + + destroy_conditionally!(token) do + token.revoke! + end end end end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 7c0fdd3d1be..da71787abab 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -88,6 +88,7 @@ module API variable = user_project.variables.find_by(key: params[:key]) not_found!('Variable') unless variable + # Variables don't have any timestamp. Therfore, destroy unconditionally. status 204 variable.destroy end -- cgit v1.2.1 From bed27b6a6d1fea22596ca8e643686c8aa61fa65d Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Mon, 28 Aug 2017 12:46:00 -0400 Subject: Update rubocop-gitlab-security to 0.1.0 --- lib/after_commit_queue.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index b67575a3ac2..4750a2c373a 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -7,7 +7,7 @@ module AfterCommitQueue end def run_after_commit(method = nil, &block) - _after_commit_queue << proc { self.send(method) } if method + _after_commit_queue << proc { self.send(method) } if method # rubocop:disable GitlabSecurity/PublicSend _after_commit_queue << block if block true end -- cgit v1.2.1 From 87b51c5981db3b1b9831b01ca6e74127d57dc2d9 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 29 Aug 2017 07:14:41 +0900 Subject: Move the logic to a concern --- lib/gitlab/sql/pattern.rb | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 46c973d8a11..26bfeeeee67 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -1,33 +1,26 @@ module Gitlab module SQL - class Pattern - MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - - attr_reader :query + module Pattern + extend ActiveSupport::Concern - def initialize(query) - @query = query - end + MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - def to_sql - if exact_matching? - sanitized_query - else - "%#{sanitized_query}%" + class_methods do + def to_pattern(query) + if exact_matching?(query) + sanitize_sql_like(query) + else + "%#{sanitize_sql_like(query)}%" + end end - end - def exact_matching? - !partial_matching? - end - - def partial_matching? - @query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING - end + def exact_matching?(query) + query.length < MIN_CHARS_FOR_PARTIAL_MATCHING + end - def sanitized_query - # Note: ActiveRecord::Base.sanitize_sql_like is a protected method - ActiveRecord::Base.__send__(:sanitize_sql_like, query) + def partial_matching?(query) + query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING + end end end end -- cgit v1.2.1 From ee4820a5268d02fb7ed142511d1a194f06629a1e Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Mon, 28 Aug 2017 17:13:22 +0200 Subject: Add a spec when ressource is not modified --- lib/api/projects.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 78e25b6da03..78d900984ac 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -365,8 +365,7 @@ module API authorize! :remove_fork_project, user_project if user_project.forked? - status 204 - user_project.forked_project_link.destroy + destroy_conditionally!(user_project.forked_project_link) else not_modified! end @@ -410,8 +409,7 @@ module API link = user_project.project_group_links.find_by(group_id: params[:group_id]) not_found!('Group Link') unless link - status 204 - link.destroy + destroy_conditionally!(link) end desc 'Upload a file' -- cgit v1.2.1 From 87467127b6e0927d56e532f4d6adc4091ff9ef6f Mon Sep 17 00:00:00 2001 From: Maxim Rydkin Date: Thu, 24 Aug 2017 19:35:06 +0300 Subject: replace `is_ancestor?` with `ancestor?` --- lib/gitlab/git/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index b835dec24eb..dce3a9b2d37 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -439,7 +439,7 @@ module Gitlab end # Returns true is +from+ is direct ancestor to +to+, otherwise false - def is_ancestor?(from, to) + def ancestor?(from, to) gitaly_commit_client.is_ancestor(from, to) end -- cgit v1.2.1 From 9226804bf32bcecd826818eb626714b52285c5e8 Mon Sep 17 00:00:00 2001 From: Maxim Rydkin Date: Thu, 24 Aug 2017 19:44:36 +0300 Subject: replace `is_runner_queue_value_latest?` with `runner_queue_value_latest?` --- lib/api/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 88fc62d33df..1a7ded31c91 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -78,7 +78,7 @@ module API no_content! unless current_runner.active? update_runner_info - if current_runner.is_runner_queue_value_latest?(params[:last_update]) + if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] Gitlab::Metrics.add_event(:build_not_found_cached) return no_content! -- cgit v1.2.1 From 6a56bec735c7434c85e3b8776b8413d3bdcb93ec Mon Sep 17 00:00:00 2001 From: Maxim Rydkin Date: Thu, 24 Aug 2017 19:58:09 +0300 Subject: replace `is_gitlab_user?` with `gitlab_user?` --- lib/system_check/app/git_config_check.rb | 2 +- lib/tasks/gitlab/task_helpers.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb index 198867f7ac6..d08a81639e3 100644 --- a/lib/system_check/app/git_config_check.rb +++ b/lib/system_check/app/git_config_check.rb @@ -20,7 +20,7 @@ module SystemCheck # Returns true if all subcommands were successful (according to their exit code) # Returns false if any or all subcommands failed. def repair! - return false unless is_gitlab_user? + return false unless gitlab_user? command_success = OPTIONS.map do |name, value| system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index d85b810ac66..8a63f486fa3 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -104,7 +104,7 @@ module Gitlab Gitlab.config.gitlab.user end - def is_gitlab_user? + def gitlab_user? return @is_gitlab_user unless @is_gitlab_user.nil? current_user = run_command(%w(whoami)).chomp @@ -114,7 +114,7 @@ module Gitlab def warn_user_is_not_gitlab return if @warned_user_not_gitlab - unless is_gitlab_user? + unless gitlab_user? current_user = run_command(%w(whoami)).chomp puts " Warning ".color(:black).background(:yellow) -- cgit v1.2.1 From 78c5d4ddc2af74f352d4871689c7b8451b612b13 Mon Sep 17 00:00:00 2001 From: Maxim Rydkin Date: Thu, 24 Aug 2017 20:01:22 +0300 Subject: replace `is_multi_check?` with `multi_check?` --- lib/system_check/base_check.rb | 2 +- lib/system_check/simple_executor.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 5dcb3f0886b..7f9e2ffffc2 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -73,7 +73,7 @@ module SystemCheck self.class.instance_methods(false).include?(:skip?) end - def is_multi_check? + def multi_check? self.class.instance_methods(false).include?(:multi_check) end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index e5986612908..6604b1078cf 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -53,7 +53,7 @@ module SystemCheck end # When implements a multi check, we don't control the output - if check.is_multi_check? + if check.multi_check? check.multi_check return end -- cgit v1.2.1 From 622c912d5b896380a0d0a1db61bbf9d1cb990c80 Mon Sep 17 00:00:00 2001 From: Maxim Rydkin Date: Thu, 24 Aug 2017 20:02:27 +0300 Subject: replace `is_successful?` with `successful?` --- lib/gitlab/health_checks/db_check.rb | 2 +- lib/gitlab/health_checks/redis/cache_check.rb | 2 +- lib/gitlab/health_checks/redis/queues_check.rb | 2 +- lib/gitlab/health_checks/redis/redis_check.rb | 2 +- lib/gitlab/health_checks/redis/shared_state_check.rb | 2 +- lib/gitlab/health_checks/simple_abstract_check.rb | 8 ++++---- 6 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index fd94984f8a2..e27e16ddaf6 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -10,7 +10,7 @@ module Gitlab 'db_ping' end - def is_successful?(result) + def successful?(result) result == '1' end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index a28658d42d4..0eb9b77634a 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_cache_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb index f97d50d3947..f322fe831b8 100644 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_queues_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index fe4e3c4a3ab..8ceb0a0aa46 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -11,7 +11,7 @@ module Gitlab 'redis_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb index e3244392902..07e6f707998 100644 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -15,7 +15,7 @@ module Gitlab 'redis_shared_state_ping' end - def is_successful?(result) + def successful?(result) result == 'PONG' end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index f5026171ba4..96945ce5b20 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -5,7 +5,7 @@ module Gitlab def readiness check_result = check - if is_successful?(check_result) + if successful?(check_result) HealthChecks::Result.new(true) elsif check_result.is_a?(Timeout::Error) HealthChecks::Result.new(false, "#{human_name} check timed out") @@ -16,10 +16,10 @@ module Gitlab def metrics result, elapsed = with_timing(&method(:check)) - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), - metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_success", successful?(result) ? 1 : 0), metric("#{metric_prefix}_latency_seconds", elapsed) ] end @@ -30,7 +30,7 @@ module Gitlab raise NotImplementedError end - def is_successful?(result) + def successful?(result) raise NotImplementedError end -- cgit v1.2.1 From e6c7c11a5bb2656c2be997368b62aca61bb1f485 Mon Sep 17 00:00:00 2001 From: Maxim Rydkin Date: Thu, 24 Aug 2017 20:16:31 +0300 Subject: replace `has_matching_label` with `has_matching_label?` --- lib/gitlab/prometheus/queries/matched_metrics_query.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index d4894c87f8d..4c3edccc71a 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -42,13 +42,13 @@ module Gitlab lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| client_series(*batched_series, start: timeframe_start, stop: timeframe_end) - .select(&method(:has_matching_label)) + .select(&method(:has_matching_label?)) .map { |series_info| [series_info['__name__'], true] } end lookup.to_h end - def has_matching_label(series_info) + def has_matching_label?(series_info) series_info.key?('environment') end -- cgit v1.2.1 From 10ae0d8316bd9ae5aa1ce46fcea3ec8f01a3b336 Mon Sep 17 00:00:00 2001 From: Maxim Rydkin Date: Thu, 24 Aug 2017 20:52:43 +0300 Subject: replace `is_ancestor` with `ancestor?` --- lib/gitlab/checks/force_push.rb | 2 +- lib/gitlab/git/repository.rb | 2 +- lib/gitlab/gitaly_client/commit_service.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 714464fd5e7..dc5d285ea65 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -12,7 +12,7 @@ module Gitlab !project .repository .gitaly_commit_client - .is_ancestor(oldrev, newrev) + .ancestor?(oldrev, newrev) else Gitlab::Git::RevList.new( path_to_repo: project.repository.path_to_repo, diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index dce3a9b2d37..03e2bec84dd 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -440,7 +440,7 @@ module Gitlab # Returns true is +from+ is direct ancestor to +to+, otherwise false def ancestor?(from, to) - gitaly_commit_client.is_ancestor(from, to) + gitaly_commit_client.ancestor?(from, to) end # Return an array of Diff objects that represent the diff diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 57f42bd35ee..21a32a7e0db 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -22,7 +22,7 @@ module Gitlab end end - def is_ancestor(ancestor_id, child_id) + def ancestor?(ancestor_id, child_id) request = Gitaly::CommitIsAncestorRequest.new( repository: @gitaly_repo, ancestor_id: ancestor_id, -- cgit v1.2.1 From 9954dafda58a0d5d856fdbbef01633e674638aa1 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 29 Aug 2017 16:23:12 +0800 Subject: Just use a block --- lib/gitlab/git/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f6d30ad7534..adf15473eb6 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -585,7 +585,7 @@ module Gitlab end def delete_refs(ref_names) - ref_names.each(&rugged.references.method(:delete)) + ref_names.each { |ref| rugged.references.delete(ref) } end # Create a new branch named **ref+ based on **stat_point+, HEAD by default -- cgit v1.2.1 From 12633b46b6884dda4ffd87b14b4b52725acd6ec1 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 29 Aug 2017 18:00:03 +0900 Subject: Refactor --- lib/gitlab/sql/pattern.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 26bfeeeee67..b42bc67ccfc 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -7,17 +7,13 @@ module Gitlab class_methods do def to_pattern(query) - if exact_matching?(query) - sanitize_sql_like(query) - else + if partial_matching?(query) "%#{sanitize_sql_like(query)}%" + else + sanitize_sql_like(query) end end - def exact_matching?(query) - query.length < MIN_CHARS_FOR_PARTIAL_MATCHING - end - def partial_matching?(query) query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end -- cgit v1.2.1 From 5eab624d3ce7500b3cc19230b5ce86125b88015b Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 18 Aug 2017 14:26:23 +0200 Subject: Improve migrations using triggers This adds a bunch of checks to migrations that may create or drop triggers. Dropping triggers/functions is done using "IF EXISTS" so we don't throw an error if the object in question has already been dropped. We now also raise a custom error (message) when the user does not have TRIGGER privileges. This should prevent the schema from entering an inconsistent state while also providing the user with enough information on how to solve the problem. The recommendation of using SUPERUSER permissions is a bit extreme but we require this anyway (Omnibus also configures users with this permission). Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/36633 --- lib/gitlab/database.rb | 8 +++++++ lib/gitlab/database/grant.rb | 34 ++++++++++++++++++++++++++++++ lib/gitlab/database/migration_helpers.rb | 36 ++++++++++++++++++++++++++++---- 3 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 lib/gitlab/database/grant.rb (limited to 'lib') diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index e001d25e7b7..a6ec75da385 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -9,6 +9,14 @@ module Gitlab ActiveRecord::Base.configurations[Rails.env] end + def self.username + config['username'] || ENV['USER'] + end + + def self.database_name + config['database'] + end + def self.adapter_name config['adapter'] end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb new file mode 100644 index 00000000000..aee3981e79a --- /dev/null +++ b/lib/gitlab/database/grant.rb @@ -0,0 +1,34 @@ +module Gitlab + module Database + # Model that can be used for querying permissions of a SQL user. + class Grant < ActiveRecord::Base + self.table_name = + if Database.postgresql? + 'information_schema.role_table_grants' + else + 'mysql.user' + end + + def self.scope_to_current_user + if Database.postgresql? + where('grantee = user') + else + where("CONCAT(User, '@', Host) = current_user()") + end + end + + # Returns true if the current user can create and execute triggers on the + # given table. + def self.create_and_execute_trigger?(table) + priv = + if Database.postgresql? + where(privilege_type: 'TRIGGER', table_name: table) + else + where(Trigger_priv: 'Y') + end + + priv.scope_to_current_user.any? + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 5e2c6cc5cad..fb14798efe6 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -358,6 +358,8 @@ module Gitlab raise 'rename_column_concurrently can not be run inside a transaction' end + check_trigger_permissions!(table) + old_col = column_for(table, old) new_type = type || old_col.type @@ -430,6 +432,8 @@ module Gitlab def cleanup_concurrent_column_rename(table, old, new) trigger_name = rename_trigger_name(table, old, new) + check_trigger_permissions!(table) + if Database.postgresql? remove_rename_triggers_for_postgresql(table, trigger_name) else @@ -485,14 +489,14 @@ module Gitlab # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) - execute("DROP TRIGGER #{trigger} ON #{table}") - execute("DROP FUNCTION #{trigger}()") + execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") + execute("DROP FUNCTION IF EXISTS #{trigger}()") end # Removes the triggers used for renaming a MySQL column concurrently. def remove_rename_triggers_for_mysql(trigger) - execute("DROP TRIGGER #{trigger}_insert") - execute("DROP TRIGGER #{trigger}_update") + execute("DROP TRIGGER IF EXISTS #{trigger}_insert") + execute("DROP TRIGGER IF EXISTS #{trigger}_update") end # Returns the (base) name to use for triggers when renaming columns. @@ -625,6 +629,30 @@ module Gitlab conn.llen("queue:#{queue_name}") end end + + def check_trigger_permissions!(table) + unless Grant.create_and_execute_trigger?(table) + dbname = Database.database_name + user = Database.username + + raise <<-EOF +Your database user is not allowed to create, drop, or execute triggers on the +table #{table}. + +If you are using PostgreSQL you can solve this by logging in to the GitLab +database (#{dbname}) using a super user and running: + + ALTER #{user} WITH SUPERUSER + +For MySQL you instead need to run: + + GRANT ALL PRIVILEGES ON *.* TO #{user}@'%' + +Both queries will grant the user super user permissions, ensuring you don't run +into similar problems in the future (e.g. when new tables are created). + EOF + end + end end end end -- cgit v1.2.1 From ce1ce82045f168143ccc143f5200ea9da820d990 Mon Sep 17 00:00:00 2001 From: Travis Miller Date: Mon, 21 Aug 2017 17:42:03 -0500 Subject: Resolve new N+1 by adding preloads and metadata to issues end points --- lib/api/entities.rb | 22 ++++++++++++++++++++-- lib/api/issues.rb | 22 +++++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e8dd61e493f..e18c69b7a3d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -320,7 +320,10 @@ module API end class IssueBasic < ProjectEntity - expose :label_names, as: :labels + expose :labels do |issue, options| + # Avoids an N+1 query since labels are preloaded + issue.labels.map(&:title).sort + end expose :milestone, using: Entities::Milestone expose :assignees, :author, using: Entities::UserBasic @@ -329,7 +332,22 @@ module API end expose :user_notes_count - expose :upvotes, :downvotes + expose :upvotes do |issue, options| + if options[:issuable_metadata] + # Avoids an N+1 query when metadata is included + options[:issuable_metadata][issue.id].upvotes + else + issue.upvotes + end + end + expose :downvotes do |issue, options| + if options[:issuable_metadata] + # Avoids an N+1 query when metadata is included + options[:issuable_metadata][issue.id].downvotes + else + issue.downvotes + end + end expose :due_date expose :confidential diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4cec1145f3a..64a425eb96e 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -4,6 +4,8 @@ module API before { authenticate! } + helpers ::Gitlab::IssuableMetadata + helpers do def find_issues(args = {}) args = params.merge(args) @@ -13,6 +15,7 @@ module API args[:label_name] = args.delete(:labels) issues = IssuesFinder.new(current_user, args).execute + .preload(:assignees, :labels, :notes) issues.reorder(args[:order_by] => args[:sort]) end @@ -65,7 +68,11 @@ module API get do issues = find_issues - present paginate(issues), with: Entities::IssueBasic, current_user: current_user + options = { with: Entities::IssueBasic, + current_user: current_user } + options[:issuable_metadata] = issuable_meta_data(issues, 'Issue') + + present paginate(issues), options end end @@ -86,7 +93,11 @@ module API issues = find_issues(group_id: group.id) - present paginate(issues), with: Entities::IssueBasic, current_user: current_user + options = { with: Entities::IssueBasic, + current_user: current_user } + options[:issuable_metadata] = issuable_meta_data(issues, 'Issue') + + present paginate(issues), options end end @@ -109,7 +120,12 @@ module API issues = find_issues(project_id: project.id) - present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project + options = { with: Entities::IssueBasic, + current_user: current_user, + project: user_project } + options[:issuable_metadata] = issuable_meta_data(issues, 'Issue') + + present paginate(issues), options end desc 'Get a single project issue' do -- cgit v1.2.1 From 749c389345cf382b740277db62f7d4b849902d60 Mon Sep 17 00:00:00 2001 From: Travis Miller Date: Mon, 28 Aug 2017 19:02:26 -0500 Subject: Add time stats to issue and merge request API end points --- lib/api/entities.rb | 22 +++++++++++++++++++++- lib/api/issues.rb | 28 +++++++++++++++++----------- lib/api/merge_requests.rb | 2 +- 3 files changed, 39 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e18c69b7a3d..803b48dd88a 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -354,6 +354,10 @@ module API expose :web_url do |issue, options| Gitlab::UrlBuilder.build(issue) end + + expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |issue| + issue + end end class Issue < IssueBasic @@ -383,10 +387,22 @@ module API end class IssuableTimeStats < Grape::Entity + format_with(:time_tracking_formatter) do |time_spent| + Gitlab::TimeTrackingFormatter.output(time_spent) + end + expose :time_estimate expose :total_time_spent expose :human_time_estimate - expose :human_total_time_spent + + with_options(format_with: :time_tracking_formatter) do + expose :total_time_spent, as: :human_total_time_spent + end + + def total_time_spent + # Avoids an N+1 query since timelogs are preloaded + object.timelogs.map(&:time_spent).sum + end end class ExternalIssue < Grape::Entity @@ -436,6 +452,10 @@ module API expose :web_url do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) end + + expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |merge_request| + merge_request + end end class MergeRequest < MergeRequestBasic diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 64a425eb96e..c30e430ae85 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -15,7 +15,7 @@ module API args[:label_name] = args.delete(:labels) issues = IssuesFinder.new(current_user, args).execute - .preload(:assignees, :labels, :notes) + .preload(:assignees, :labels, :notes, :timelogs) issues.reorder(args[:order_by] => args[:sort]) end @@ -68,9 +68,11 @@ module API get do issues = find_issues - options = { with: Entities::IssueBasic, - current_user: current_user } - options[:issuable_metadata] = issuable_meta_data(issues, 'Issue') + options = { + with: Entities::IssueBasic, + current_user: current_user, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } present paginate(issues), options end @@ -93,9 +95,11 @@ module API issues = find_issues(group_id: group.id) - options = { with: Entities::IssueBasic, - current_user: current_user } - options[:issuable_metadata] = issuable_meta_data(issues, 'Issue') + options = { + with: Entities::IssueBasic, + current_user: current_user, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } present paginate(issues), options end @@ -120,10 +124,12 @@ module API issues = find_issues(project_id: project.id) - options = { with: Entities::IssueBasic, - current_user: current_user, - project: user_project } - options[:issuable_metadata] = issuable_meta_data(issues, 'Issue') + options = { + with: Entities::IssueBasic, + current_user: current_user, + project: user_project, + issuable_metadata: issuable_meta_data(issues, 'Issue') + } present paginate(issues), options end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 8810d4e441d..a3284afb745 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -21,7 +21,7 @@ module API return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) + .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels, :timelogs) end params :merge_requests_params do -- cgit v1.2.1 From 67c042e4a5603a39494c3c7e407161348d7e85f3 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Tue, 29 Aug 2017 16:49:43 +0200 Subject: Respect the default visibility level when creating a group --- lib/api/groups.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e56427304a6..15266e9d4e5 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,7 +7,9 @@ module API helpers do params :optional_params_ce do optional :description, type: String, desc: 'The description of the group' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, + default: Gitlab::VisibilityLevel.string_level(Gitlab::CurrentSettings.current_application_settings.default_group_visibility), + desc: 'The visibility of the group' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' -- cgit v1.2.1 From 9edaff0d3547ba9854e69655a4ae4d5095b8a25f Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Wed, 30 Aug 2017 10:11:24 +0200 Subject: Make rubocop happy --- lib/api/groups.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 15266e9d4e5..cf3cee0073c 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,9 +7,11 @@ module API helpers do params :optional_params_ce do optional :description, type: String, desc: 'The description of the group' - optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, - default: Gitlab::VisibilityLevel.string_level(Gitlab::CurrentSettings.current_application_settings.default_group_visibility), - desc: 'The visibility of the group' + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + default: Gitlab::VisibilityLevel.string_level( + Gitlab::CurrentSettings.current_application_settings.default_group_visibility), + desc: 'The visibility of the group' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' -- cgit v1.2.1 From 808fb2549bf8b3dee58a4af81a4b6513a5016342 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 30 Aug 2017 13:20:55 +0200 Subject: Raise exception when simplifiable ci entry incomplete --- lib/gitlab/ci/config/entry/simplifiable.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb index c26576fcbcd..0a03027736f 100644 --- a/lib/gitlab/ci/config/entry/simplifiable.rb +++ b/lib/gitlab/ci/config/entry/simplifiable.rb @@ -6,6 +6,10 @@ module Gitlab EntryStrategy = Struct.new(:name, :condition) def initialize(config, **metadata) + unless self.class.const_defined?(:UnknownStrategy) + raise ArgumentError, 'UndefinedStrategy not available!' + end + strategy = self.class.strategies.find do |variant| variant.condition.call(config) end -- cgit v1.2.1 From 3474e109b73e0373a2663b5ae06db9aab61e1c02 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 30 Aug 2017 13:27:48 +0200 Subject: Simplify code for appending strategies in CI/CD config --- lib/gitlab/ci/config/entry/simplifiable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb index 0a03027736f..12764629686 100644 --- a/lib/gitlab/ci/config/entry/simplifiable.rb +++ b/lib/gitlab/ci/config/entry/simplifiable.rb @@ -21,12 +21,12 @@ module Gitlab def self.strategy(name, **opts) EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| - (@strategies ||= []).append(strategy) + strategies.append(strategy) end end def self.strategies - @strategies.to_a + @strategies ||= [] end def self.entry_class(strategy) -- cgit v1.2.1 From 1bf87d25241eb5e1a0f147d350814a3a1cfa1343 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 30 Aug 2017 14:47:54 +0200 Subject: Remove unused expressions policy from ci/cd config --- lib/gitlab/ci/config/entry/policy.rb | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 05602f1b3c6..3cdae1cee4f 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -7,7 +7,6 @@ module Gitlab # class Policy < Simplifiable strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } - strategy :ExpressionsPolicy, if: -> (config) { config.is_a?(Hash) } class RefsPolicy < Entry::Node include Entry::Validatable @@ -17,24 +16,6 @@ module Gitlab end end - class ExpressionsPolicy < Entry::Node - include Entry::Validatable - include Entry::Attributable - - attributes :refs, :expressions - - validations do - validates :config, presence: true - validates :config, allowed_keys: %i[refs expressions] - - with_options allow_nil: true do - validates :refs, array_of_strings_or_regexps: true - validates :expressions, type: Array - validates :expressions, presence: true - end - end - end - class UnknownStrategy < Entry::Node def errors ["#{location} has to be either an array of conditions or a hash"] -- cgit v1.2.1 From b9d8946395ebe2b0d05e464e6a2af1181c7f1b90 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Wed, 30 Aug 2017 13:33:39 +0100 Subject: Don't use public_send in destroy_conditionally! helper As we only override in two places, we could just ask for the value rather than the method name. --- lib/api/branches.rb | 2 +- lib/api/helpers.rb | 6 ++++-- lib/api/tags.rb | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/branches.rb b/lib/api/branches.rb index b87f7cdbad1..a989394ad91 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -130,7 +130,7 @@ module API commit = user_project.repository.commit(branch.dereferenced_target) - destroy_conditionally!(commit, last_update_field: :authored_date) do + destroy_conditionally!(commit, last_updated: commit.authored_date) do result = DeleteBranchService.new(user_project, current_user) .execute(params[:branch]) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 84980864151..3d377fdb9eb 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -19,8 +19,10 @@ module API end end - def destroy_conditionally!(resource, last_update_field: :updated_at) - check_unmodified_since!(resource.public_send(last_update_field)) + def destroy_conditionally!(resource, last_updated: nil) + last_updated ||= resource.updated_at + + check_unmodified_since!(last_updated) status 204 if block_given? diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 81b17935b81..912415e3a7f 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -70,7 +70,7 @@ module API commit = user_project.repository.commit(tag.dereferenced_target) - destroy_conditionally!(commit, last_update_field: :authored_date) do + destroy_conditionally!(commit, last_updated: commit.authored_date) do result = ::Tags::DestroyService.new(user_project, current_user) .execute(params[:tag_name]) -- cgit v1.2.1 From c5553ce772371295d2d7652cec899633042fae07 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Wed, 30 Aug 2017 21:15:06 +0800 Subject: Use `git update-ref --stdin -z` to delete refs --- lib/gitlab/git/repository.rb | 18 ++++++++++++++++-- lib/tasks/gitlab/cleanup.rake | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 8b39e92cc65..fb6504bdea0 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -17,6 +17,7 @@ module Gitlab NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) InvalidRef = Class.new(StandardError) + GitError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the storage path, not the storage name @@ -598,8 +599,21 @@ module Gitlab rugged.branches.delete(branch_name) end - def delete_refs(ref_names) - ref_names.each { |ref| rugged.references.delete(ref) } + def delete_refs(*ref_names) + instructions = ref_names.map do |ref| + "delete #{ref}\x00\x00" + end + + command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] + message, status = Gitlab::Popen.popen( + command, + path) do |stdin| + stdin.write(instructions.join) + end + + unless status.zero? + raise GitError.new("Could not delete refs #{ref_names}: #{message}") + end end # Create a new branch named **ref+ based on **stat_point+, HEAD by default diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index f76bef5f4bf..8ae1b6a626a 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -111,7 +111,7 @@ namespace :gitlab do next unless id > max_iid project.deployments.find(id).create_ref - rugged.references.delete(ref) + project.repository.delete_refs(ref) end end end -- cgit v1.2.1 From 59e5393827e9e9eddb9bb0a960f1cda1f6d9511d Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 23 Aug 2017 19:54:14 +0900 Subject: Fuzzy search issuable title or description --- lib/gitlab/sql/pattern.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index b42bc67ccfc..7c2d1d8f887 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -4,6 +4,7 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 + REGEX_QUOTED_WORD = /(?<=^| )"[^"]+"(?= |$)/ class_methods do def to_pattern(query) @@ -17,6 +18,28 @@ module Gitlab def partial_matching?(query) query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end + + def to_fuzzy_arel(column, query) + words = select_fuzzy_words(query) + + matches = words.map { |word| arel_table[column].matches(to_pattern(word)) } + + matches.reduce { |result, match| result.and(match) } + end + + def select_fuzzy_words(query) + quoted_words = query.scan(REGEX_QUOTED_WORD) + + query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } + + words = query.split(/\s+/) + + quoted_words.map! { |quoted_word| quoted_word[1..-2] } + + words.concat(quoted_words) + + words.select { |word| partial_matching?(word) } + end end end end -- cgit v1.2.1 From f3b6c552f6763e20bbccaa80e306d939d6cf602c Mon Sep 17 00:00:00 2001 From: Marc Siegfriedt Date: Mon, 21 Aug 2017 21:09:45 +0000 Subject: fix :file_path - add requirements: --- lib/api/files.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/files.rb b/lib/api/files.rb index e2ac7142bc4..1598d3c00b8 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,5 +1,7 @@ module API class Files < Grape::API + FILE_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) + # Prevents returning plain/text responses for files with .txt extension after_validation { content_type "application/json" } @@ -58,13 +60,13 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag commit' end - get ":id/repository/files/:file_path/raw" do + get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! send_git_blob @repo, @blob @@ -75,7 +77,7 @@ module API requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit' end - get ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! { @@ -95,7 +97,7 @@ module API params do use :extended_file_params end - post ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + post ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -113,7 +115,7 @@ module API params do use :extended_file_params end - put ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + put ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -137,7 +139,7 @@ module API params do use :simple_file_params end - delete ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + delete ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project file_params = declared_params(include_missing: false) -- cgit v1.2.1 From b77176d11a7a31acd38b05aa39afc9ebed5a3915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Thu, 10 Aug 2017 16:08:48 +0200 Subject: Migrate Repository.FetchRemote to Gitaly - `Gitlab::Shell.fetch_remote` now takes a `Gitlab::Git::Repository` instead --- lib/gitlab/git/repository.rb | 3 ++ lib/gitlab/gitaly_client/repository_service.rb | 16 ++++++++ lib/gitlab/shell.rb | 55 +++++++++++++++++--------- 3 files changed, 56 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index b835dec24eb..ec923c990ae 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -46,6 +46,9 @@ module Gitlab # Directory name of repo attr_reader :name + # Relative path of repo + attr_reader :relative_path + # Rugged repo object attr_reader :rugged diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index a74a6dc6e78..177a1284f38 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -37,6 +37,22 @@ module Gitlab request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: revision) GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end + + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false) + request = Gitaly::FetchRemoteRequest.new(repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags) + + if ssh_auth&.ssh_import? + if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? + request.ssh_key = ssh_auth.ssh_private_key + end + + if ssh_auth.ssh_known_hosts.present? + request.known_hosts = ssh_auth.ssh_known_hosts + end + end + + GitalyClient.call(@storage, :repository_service, :fetch_remote, request) + end end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 280a9abf03e..81ecdf43ef9 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -98,33 +98,24 @@ module Gitlab # Fetch remote for repository # - # name - project path with namespace + # repository - an instance of Git::Repository # remote - remote name # forced - should we use --force flag? # no_tags - should we use --no-tags flag? # # Ex. - # fetch_remote("gitlab/gitlab-ci", "upstream") + # fetch_remote(my_repo, "upstream") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 - def fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) - args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] - args << '--force' if forced - args << '--no-tags' if no_tags - - vars = {} - - if ssh_auth&.ssh_import? - if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? - vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key - end - - if ssh_auth.ssh_known_hosts.present? - vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts + def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) + gitaly_migrate(:fetch_remote) do |is_enabled| + if is_enabled + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + else + storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] + local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) end end - - gitlab_shell_fast_execute_raise_error(args, vars) end # Move repository @@ -302,6 +293,26 @@ module Gitlab private + def local_fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false) + args = [gitlab_shell_projects_path, 'fetch-remote', storage, name, remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] + args << '--force' if forced + args << '--no-tags' if no_tags + + vars = {} + + if ssh_auth&.ssh_import? + if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? + vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key + end + + if ssh_auth.ssh_known_hosts.present? + vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts + end + end + + gitlab_shell_fast_execute_raise_error(args, vars) + end + def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -325,5 +336,13 @@ module Gitlab # from wasting I/O by searching through GEM_PATH Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end + + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound, GRPC::BadStatus => e + # Old Popen code returns [Error, output] to the caller, so we + # need to do the same here... + raise Error, e + end end end -- cgit v1.2.1 From b0f982fbdf69c292ab4530c0aaaf1ab42f4e7a01 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Mon, 21 Aug 2017 11:30:03 +0100 Subject: Add settings for minimum key strength and allowed key type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is an amalgamation of: * Cory Hinshaw: Initial implementation !5552 * Rémy Coutable: Updates !9350 * Nick Thomas: Resolve conflicts and add ED25519 support !13712 --- lib/api/entities.rb | 1 + lib/api/settings.rb | 6 ++++ lib/gitlab/git_access.rb | 9 +++++ lib/gitlab/key_fingerprint.rb | 48 ------------------------- lib/gitlab/ssh_public_key.rb | 84 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 48 deletions(-) delete mode 100644 lib/gitlab/key_fingerprint.rb create mode 100644 lib/gitlab/ssh_public_key.rb (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 803b48dd88a..8f766ba4f8d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -744,6 +744,7 @@ module API expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) } expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) } expose :password_authentication_enabled, as: :signin_enabled + expose :allowed_key_types end class Release < Grape::Entity diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 667ba468ce6..6ace0e1e390 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -122,6 +122,12 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + optional :minimum_rsa_bits, type: Integer, values: Gitlab::SSHPublicKey.allowed_sizes('rsa'), desc: 'The minimum allowed bit length of an uploaded RSA key.' + optional :minimum_dsa_bits, type: Integer, values: Gitlab::SSHPublicKey.allowed_sizes('dsa'), desc: 'The minimum allowed bit length of an uploaded DSA key.' + optional :minimum_ecdsa_bits, type: Integer, values: Gitlab::SSHPublicKey.allowed_sizes('ecdsa'), desc: 'The minimum allowed curve size (in bits) of an uploaded ECDSA key.' + optional :minimum_ed25519_bits, type: Integer, values: Gitlab::SSHPublicKey.allowed_sizes('ed25519'), desc: 'The minimum allowed curve size (in bits) of an uploaded ED25519 key.' + optional :allowed_key_types, type: Array[String], values: Gitlab::SSHPublicKey.technology_names, desc: 'The SSH key types accepted by the application (`rsa`, `dsa`, `ecdsa` or `ed25519`).' + optional(*::ApplicationSettingsHelper.visible_attributes) at_least_one_of(*::ApplicationSettingsHelper.visible_attributes) end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 3e8b83c0f90..9eab26d111e 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -34,6 +34,7 @@ module Gitlab end def check(cmd, changes) + check_valid_actor! check_protocol! check_active_user! check_project_accessibility! @@ -70,6 +71,14 @@ module Gitlab private + def check_valid_actor! + return unless actor.is_a?(Key) + + unless actor.valid? + raise UnauthorizedError, "Your SSH key #{actor.errors[:key].first}." + end + end + def check_protocol! unless protocol_allowed? raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb deleted file mode 100644 index d9a79f7c291..00000000000 --- a/lib/gitlab/key_fingerprint.rb +++ /dev/null @@ -1,48 +0,0 @@ -module Gitlab - class KeyFingerprint - attr_reader :key, :ssh_key - - # Unqualified MD5 fingerprint for compatibility - delegate :fingerprint, to: :ssh_key, allow_nil: true - - def initialize(key) - @key = key - - @ssh_key = - begin - Net::SSH::KeyFactory.load_data_public_key(key) - rescue Net::SSH::Exception, NotImplementedError - end - end - - def valid? - ssh_key.present? - end - - def type - return unless valid? - - parts = ssh_key.ssh_type.split('-') - parts.shift if parts[0] == 'ssh' - - parts[0].upcase - end - - def bits - return unless valid? - - case type - when 'RSA' - ssh_key.n.num_bits - when 'DSS', 'DSA' - ssh_key.p.num_bits - when 'ECDSA' - ssh_key.group.order.num_bits - when 'ED25519' - 256 - else - raise "Unsupported key type: #{type}" - end - end - end -end diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb new file mode 100644 index 00000000000..2df31bcc246 --- /dev/null +++ b/lib/gitlab/ssh_public_key.rb @@ -0,0 +1,84 @@ +module Gitlab + class SSHPublicKey + TYPES = %w[rsa dsa ecdsa ed25519].freeze + + Technology = Struct.new(:name, :allowed_sizes) + + Technologies = [ + Technology.new('rsa', [1024, 2048, 3072, 4096]), + Technology.new('dsa', [1024, 2048, 3072]), + Technology.new('ecdsa', [256, 384, 521]), + Technology.new('ed25519', [256]) + ].freeze + + def self.technology_names + Technologies.map(&:name) + end + + def self.technology(name) + Technologies.find { |ssh_key_technology| ssh_key_technology.name == name } + end + private_class_method :technology + + def self.allowed_sizes(name) + technology(name).allowed_sizes + end + + def self.allowed_type?(type) + technology_names.include?(type.to_s) + end + + attr_reader :key_text, :key + + # Unqualified MD5 fingerprint for compatibility + delegate :fingerprint, to: :key, allow_nil: true + + def initialize(key_text) + @key_text = key_text + + @key = + begin + Net::SSH::KeyFactory.load_data_public_key(key_text) + rescue StandardError, NotImplementedError + end + end + + def valid? + key.present? + end + + def type + return unless valid? + + case key + when OpenSSL::PKey::EC + :ecdsa + when OpenSSL::PKey::RSA + :rsa + when OpenSSL::PKey::DSA + :dsa + when Net::SSH::Authentication::ED25519::PubKey + :ed25519 + else + raise "Unsupported key type: #{key.class}" + end + end + + def bits + return unless valid? + + case type + when :rsa + key.n.num_bits + when :dsa + key.p.num_bits + when :ecdsa + key.group.order.num_bits + when :ed25519 + 256 + else + raise "Unsupported key type: #{type}" + end + end + end +end -- cgit v1.2.1 From 6847060266792471c9c14518a5106e0f622cd6c5 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Fri, 25 Aug 2017 14:08:48 +0100 Subject: Rework the permissions model for SSH key restrictions `allowed_key_types` is removed and the `minimum__bits` fields are renamed to `_key_restriction`. A special sentinel value (`-1`) signifies that the key type is disabled. This also feeds through to the UI - checkboxes per key type are out, inline selection of "forbidden" and "allowed" (i.e., no restrictions) are in. As with the previous model, unknown key types are disallowed, even if the underlying ssh daemon happens to support them. The defaults have also been changed from the lowest known bit size to "no restriction". So if someone does happen to have a 768-bit RSA key, it will continue to work on upgrade, at least until the administrator restricts them. --- lib/api/entities.rb | 1 - lib/api/settings.rb | 11 +++++----- lib/gitlab/ssh_public_key.rb | 52 ++++++++++++++++++-------------------------- 3 files changed, 27 insertions(+), 37 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 8f766ba4f8d..803b48dd88a 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -744,7 +744,6 @@ module API expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) } expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) } expose :password_authentication_enabled, as: :signin_enabled - expose :allowed_key_types end class Release < Grape::Entity diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 6ace0e1e390..01123e45ee0 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -122,11 +122,12 @@ module API optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' - optional :minimum_rsa_bits, type: Integer, values: Gitlab::SSHPublicKey.allowed_sizes('rsa'), desc: 'The minimum allowed bit length of an uploaded RSA key.' - optional :minimum_dsa_bits, type: Integer, values: Gitlab::SSHPublicKey.allowed_sizes('dsa'), desc: 'The minimum allowed bit length of an uploaded DSA key.' - optional :minimum_ecdsa_bits, type: Integer, values: Gitlab::SSHPublicKey.allowed_sizes('ecdsa'), desc: 'The minimum allowed curve size (in bits) of an uploaded ECDSA key.' - optional :minimum_ed25519_bits, type: Integer, values: Gitlab::SSHPublicKey.allowed_sizes('ed25519'), desc: 'The minimum allowed curve size (in bits) of an uploaded ED25519 key.' - optional :allowed_key_types, type: Array[String], values: Gitlab::SSHPublicKey.technology_names, desc: 'The SSH key types accepted by the application (`rsa`, `dsa`, `ecdsa` or `ed25519`).' + ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| + optional :"#{type}_key_restriction", + type: Integer, + values: ApplicationSetting.supported_key_restrictions(type), + desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys." + end optional(*::ApplicationSettingsHelper.visible_attributes) at_least_one_of(*::ApplicationSettingsHelper.visible_attributes) diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 2df31bcc246..a3f8730fb04 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -1,31 +1,20 @@ module Gitlab class SSHPublicKey - TYPES = %w[rsa dsa ecdsa ed25519].freeze - - Technology = Struct.new(:name, :allowed_sizes) + Technology = Struct.new(:name, :key_class, :supported_sizes) Technologies = [ - Technology.new('rsa', [1024, 2048, 3072, 4096]), - Technology.new('dsa', [1024, 2048, 3072]), - Technology.new('ecdsa', [256, 384, 521]), - Technology.new('ed25519', [256]) + Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096]), + Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072]), + Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521]), + Technology.new(:ed25519, Net::SSH::Authentication::ED25519::PubKey, [256]) ].freeze - def self.technology_names - Technologies.map(&:name) - end - def self.technology(name) - Technologies.find { |ssh_key_technology| ssh_key_technology.name == name } + Technologies.find { |tech| tech.name.to_s == name.to_s } end - private_class_method :technology - def self.allowed_sizes(name) - technology(name).allowed_sizes - end - - def self.allowed_type?(type) - technology_names.include?(type.to_s) + def self.supported_sizes(name) + technology(name)&.supported_sizes end attr_reader :key_text, :key @@ -50,18 +39,7 @@ module Gitlab def type return unless valid? - case key - when OpenSSL::PKey::EC - :ecdsa - when OpenSSL::PKey::RSA - :rsa - when OpenSSL::PKey::DSA - :dsa - when Net::SSH::Authentication::ED25519::PubKey - :ed25519 - else - raise "Unsupported key type: #{key.class}" - end + technology.name end def bits @@ -80,5 +58,17 @@ module Gitlab raise "Unsupported key type: #{type}" end end + + private + + def technology + @technology ||= + begin + tech = Technologies.find { |tech| key.is_a?(tech.key_class) } + raise "Unsupported key type: #{key.class}" unless tech + + tech + end + end end end -- cgit v1.2.1 From b84ca08e351fc9238bef4e6b4bf74158d25d4f1d Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Mon, 28 Aug 2017 21:33:35 +0100 Subject: Address review comments --- lib/gitlab/git_access.rb | 2 +- lib/gitlab/ssh_public_key.rb | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 9eab26d111e..62d1ecae676 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -34,8 +34,8 @@ module Gitlab end def check(cmd, changes) - check_valid_actor! check_protocol! + check_valid_actor! check_active_user! check_project_accessibility! check_project_moved! diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index a3f8730fb04..89ca1298120 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -13,6 +13,10 @@ module Gitlab Technologies.find { |tech| tech.name.to_s == name.to_s } end + def self.technology_for_key(key) + Technologies.find { |tech| key.is_a?(tech.key_class) } + end + def self.supported_sizes(name) technology(name)&.supported_sizes end @@ -37,9 +41,7 @@ module Gitlab end def type - return unless valid? - - technology.name + technology.name if valid? end def bits @@ -63,12 +65,7 @@ module Gitlab def technology @technology ||= - begin - tech = Technologies.find { |tech| key.is_a?(tech.key_class) } - raise "Unsupported key type: #{key.class}" unless tech - - tech - end + self.class.technology_for_key(key) || raise("Unsupported key type: #{key.class}") end end end -- cgit v1.2.1 From eb05bdc6f589f6f0713df12582eb9f18fc4022b3 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Mon, 28 Aug 2017 21:58:36 +0100 Subject: Move the key restriction validation to its own class --- lib/api/settings.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 01123e45ee0..851b226e9e5 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -125,7 +125,7 @@ module API ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", type: Integer, - values: ApplicationSetting.supported_key_restrictions(type), + values: KeyRestrictionValidator.supported_key_restrictions(type), desc: "Restrictions on the complexity of uploaded #{type.upcase} keys. A value of #{ApplicationSetting::FORBIDDEN_KEY_VALUE} disables all #{type.upcase} keys." end -- cgit v1.2.1 From d807d3810f233f802877f66aed9de05106e62cd5 Mon Sep 17 00:00:00 2001 From: Maxim Rydkin Date: Thu, 31 Aug 2017 10:58:05 +0000 Subject: Decrease ABC threshold to 55.25 --- lib/github/import.rb | 82 +++++++++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 40 deletions(-) (limited to 'lib') diff --git a/lib/github/import.rb b/lib/github/import.rb index 7b848081e85..9354e142d3d 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -226,49 +226,51 @@ module Github while url response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - response.body.each do |raw| - representation = Github::Representation::Issue.new(raw, options) + response.body.each { |raw| populate_issue(raw) } - begin - # Every pull request is an issue, but not every issue - # is a pull request. For this reason, "shared" actions - # for both features, like manipulating assignees, labels - # and milestones, are provided within the Issues API. - if representation.pull_request? - next unless representation.has_labels? - - merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) - else - next if Issue.where(iid: representation.iid, project_id: project.id).exists? - - author_id = user_id(representation.author, project.creator_id) - issue = Issue.new - issue.iid = representation.iid - issue.project_id = project.id - issue.title = representation.title - issue.description = format_description(representation.description, representation.author) - issue.state = representation.state - issue.label_ids = label_ids(representation.labels) - issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = author_id - issue.assignee_ids = [user_id(representation.assignee)] - issue.created_at = representation.created_at - issue.updated_at = representation.updated_at - issue.save!(validate: false) - - # Fetch comments - if representation.has_comments? - comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" - fetch_comments(issue, :comment, comments_url) - end - end - rescue => e - error(:issue, representation.url, e.message) + url = response.rels[:next] + end + end + + def populate_issue(raw) + representation = Github::Representation::Issue.new(raw, options) + + begin + # Every pull request is an issue, but not every issue + # is a pull request. For this reason, "shared" actions + # for both features, like manipulating assignees, labels + # and milestones, are provided within the Issues API. + if representation.pull_request? + return unless representation.has_labels? + + merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + else + return if Issue.where(iid: representation.iid, project_id: project.id).exists? + + author_id = user_id(representation.author, project.creator_id) + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = format_description(representation.description, representation.author) + issue.state = representation.state + issue.label_ids = label_ids(representation.labels) + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = author_id + issue.assignee_ids = [user_id(representation.assignee)] + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save!(validate: false) + + # Fetch comments + if representation.has_comments? + comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" + fetch_comments(issue, :comment, comments_url) end end - - url = response.rels[:next] + rescue => e + error(:issue, representation.url, e.message) end end -- cgit v1.2.1 From 9eabd7230da59708981a9ab5f176beaf97aa6923 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 13:48:23 +0200 Subject: Remove unused method from CI/CD YAML processor --- lib/ci/gitlab_ci_yaml_processor.rb | 6 ------ 1 file changed, 6 deletions(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 72a38e97648..e767ffd2211 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -32,12 +32,6 @@ module Ci end end - def builds_for_ref(ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).map do |name, _| - build_attributes(name) - end - end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) -- cgit v1.2.1 From 2339306df6e642dfb2c8423e69a1f67a064a2119 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 13:48:42 +0200 Subject: Improve matching kubernetes job in CI/CD processor --- lib/ci/gitlab_ci_yaml_processor.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index e767ffd2211..8dff879018f 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -48,9 +48,9 @@ module Ci only_kubernetes = job.dig(:only, :kubernetes) except_kubernetes = job.dig(:except, :kubernetes) - [!only_kubernetes & !except_kubernetes, - only_kubernetes & has_kubernetes, - except_kubernetes & !has_kubernetes].any? + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? end end -- cgit v1.2.1 From 5623cb9a6eb0075306b1cecfd336c2b5ce997f19 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 13:50:52 +0200 Subject: Make some methods in CI/CD YAML processor private --- lib/ci/gitlab_ci_yaml_processor.rb | 56 ++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 8dff879018f..5676618f7fd 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,17 +20,6 @@ module Ci raise ValidationError, e.message end - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| @@ -38,22 +27,6 @@ module Ci end end - def pipeline_stage_builds(stage, pipeline) - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) - - builds.select do |build| - job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_available? - only_kubernetes = job.dig(:only, :kubernetes) - except_kubernetes = job.dig(:except, :kubernetes) - - [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? - end - end - def builds @jobs.map do |name, _| build_attributes(name) @@ -110,6 +83,35 @@ module Ci private + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_available? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? + end + end + + + def jobs_for_ref(ref, tag = false, source = nil) + @jobs.select do |_, job| + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + end + end + + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| + job[:stage] == stage + end + end + def initial_parsing ## # Global config -- cgit v1.2.1 From b6646e778d43a9b6ba768a8799830095cfcff635 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 20 Jul 2017 08:53:57 +0200 Subject: Track the locale in Sentry so we know which ones are failing --- lib/gitlab/sentry.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 2442c2ded3b..d7e73f30abf 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -7,6 +7,8 @@ module Gitlab def self.context(current_user = nil) return unless self.enabled? + Raven.tags_context(locale: I18n.locale) + if current_user Raven.user_context( id: current_user.id, -- cgit v1.2.1 From bde39322f1b0a24b03c949abbf34b21859f9a5c0 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 20 Jul 2017 17:32:17 +0200 Subject: Add a linter for PO files --- lib/gitlab/po_linter.rb | 168 ++++++++++++++++++++++++++++++++++++++++++++++++ lib/gitlab/utils.rb | 4 ++ lib/tasks/gettext.rake | 40 ++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 lib/gitlab/po_linter.rb (limited to 'lib') diff --git a/lib/gitlab/po_linter.rb b/lib/gitlab/po_linter.rb new file mode 100644 index 00000000000..54594949711 --- /dev/null +++ b/lib/gitlab/po_linter.rb @@ -0,0 +1,168 @@ +require 'simple_po_parser' + +module Gitlab + class PoLinter + attr_reader :po_path, :entries, :locale + + VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze + + def initialize(po_path, locale = I18n.locale.to_s) + @po_path = po_path + @locale = locale + end + + def errors + @errors ||= validate_po + end + + def validate_po + if parse_error = parse_po + return 'PO-syntax errors' => [parse_error] + end + + validate_entries + end + + def parse_po + @entries = SimplePoParser.parse(po_path) + nil + rescue SimplePoParser::ParserError => e + @entries = [] + e.message + end + + def validate_entries + errors = {} + + entries.each do |entry| + # Skip validation of metadata + next if entry[:msgid].empty? + + errors_for_entry = validate_entry(entry) + errors[join_message(entry[:msgid])] = errors_for_entry if errors_for_entry.any? + end + + errors + end + + def validate_entry(entry) + errors = [] + + validate_flags(errors, entry) + validate_variables(errors, entry) + validate_newlines(errors, entry) + + errors + end + + def validate_newlines(errors, entry) + message_id = join_message(entry[:msgid]) + + if entry[:msgid].is_a?(Array) + errors << "<#{message_id}> is defined over multiple lines, this breaks some tooling." + end + end + + def validate_variables(errors, entry) + if entry[:msgid_plural].present? + validate_variables_in_message(errors, entry[:msgid], entry['msgstr[0]']) + validate_variables_in_message(errors, entry[:msgid_plural], entry['msgstr[1]']) + else + validate_variables_in_message(errors, entry[:msgid], entry[:msgstr]) + end + end + + def validate_variables_in_message(errors, message_id, message_translation) + message_id = join_message(message_id) + required_variables = message_id.scan(VARIABLE_REGEX) + + validate_unnamed_variables(errors, required_variables) + validate_translation(errors, message_id, required_variables) + + message_translation = join_message(message_translation) + unless message_translation.empty? + validate_variable_usage(errors, message_translation, required_variables) + end + end + + def validate_translation(errors, message_id, used_variables) + variables = fill_in_variables(used_variables) + + begin + Gitlab::I18n.with_locale(locale) do + translated = if message_id.include?('|') + FastGettext::Translation.s_(message_id) + else + FastGettext::Translation._(message_id) + end + + translated % variables + end + + # `sprintf` could raise an `ArgumentError` when invalid passing something + # other than a Hash when using named variables + # + # `sprintf` could raise `TypeError` when passing a wrong type when using + # unnamed variables + # + # FastGettext::Translation could raise `RuntimeError` (raised as a string), + # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` + # + # `FastGettext::Translation` could raise `ArgumentError` as subclassess + # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` + rescue ArgumentError, TypeError, RuntimeError => e + errors << "Failure translating to #{locale} with #{variables}: #{e.message}" + end + end + + def fill_in_variables(variables) + if variables.empty? + [] + elsif variables.any? { |variable| unnamed_variable?(variable) } + variables.map do |variable| + variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string + end + else + variables.inject({}) do |hash, variable| + variable_name = variable[/\w+/] + hash[variable_name] = Gitlab::Utils.random_string + hash + end + end + end + + def validate_unnamed_variables(errors, variables) + if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } + errors << 'is combining multiple unnamed variables' + end + end + + def validate_variable_usage(errors, translation, required_variables) + found_variables = translation.scan(VARIABLE_REGEX) + + missing_variables = required_variables - found_variables + if missing_variables.any? + errors << "<#{translation}> is missing: [#{missing_variables.to_sentence}]" + end + + unknown_variables = found_variables - required_variables + if unknown_variables.any? + errors << "<#{translation}> is using unknown variables: [#{unknown_variables.to_sentence}]" + end + end + + def unnamed_variable?(variable_name) + !variable_name.start_with?('%{') + end + + def validate_flags(errors, entry) + if flag = entry[:flag] + errors << "is marked #{flag}" + end + end + + def join_message(message) + Array(message).join + end + end +end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 9670c93759e..abb3d3a02c3 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -42,5 +42,9 @@ module Gitlab 'No' end end + + def random_string + Random.rand(Float::MAX.to_i).to_s(36) + end end end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index b48e4dce445..b75da6bf2fc 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -19,4 +19,44 @@ namespace :gettext do Rake::Task['gettext:pack'].invoke Rake::Task['gettext:po_to_json'].invoke end + + desc 'Lint all po files in `locale/' + task lint: :environment do + FastGettext.silence_errors + files = Dir.glob(Rails.root.join('locale/*/gitlab.po')) + + linters = files.map do |file| + locale = File.basename(File.dirname(file)) + + Gitlab::PoLinter.new(file, locale) + end + + pot_file = Rails.root.join('locale/gitlab.pot') + linters.unshift(Gitlab::PoLinter.new(pot_file)) + + failed_linters = linters.select { |linter| linter.errors.any? } + + if failed_linters.empty? + puts 'All PO files are valid.' + else + failed_linters.each do |linter| + report_errors_for_file(linter.po_path, linter.errors) + end + + raise "Not all PO-files are valid: #{failed_linters.map(&:po_path).to_sentence}" + end + end + + def report_errors_for_file(file, errors_for_file) + puts "Errors in `#{file}`:" + + errors_for_file.each do |message_id, errors| + puts " #{message_id}" + errors.each do |error| + spaces = ' ' * 4 + error = error.lines.join("#{spaces}") + puts "#{spaces}#{error}" + end + end + end end -- cgit v1.2.1 From 5883ce95efcc4cc04f949f9b4e66d73fbede94e2 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Thu, 31 Aug 2017 10:47:03 +0100 Subject: `current_application_settings` belongs on `Gitlab::CurrentSettings` The initializers including this were doing so at the top level, so every object loaded after them had a `current_application_settings` method. However, if someone had rack-attack enabled (which was loaded before these initializers), it would try to load the API, and fail, because `Gitlab::CurrentSettings` didn't have that method. To fix this: 1. Don't include `Gitlab::CurrentSettings` at the top level. We do not need `Object.new.current_application_settings` to work. 2. Make `Gitlab::CurrentSettings` explicitly `extend self`, as we already use it like that in several places. 3. Change the initializers to use that new form. --- lib/api/helpers/runner.rb | 2 ++ lib/email_template_interceptor.rb | 2 +- lib/gitlab/asciidoc.rb | 2 ++ lib/gitlab/auth.rb | 2 ++ lib/gitlab/current_settings.rb | 2 ++ lib/gitlab/gon_helper.rb | 1 + lib/gitlab/metrics/influx_db.rb | 2 +- lib/gitlab/performance_bar.rb | 2 +- lib/gitlab/polling_interval.rb | 2 +- lib/gitlab/protocol_access.rb | 2 ++ lib/gitlab/recaptcha.rb | 2 ++ lib/gitlab/sentry.rb | 2 ++ lib/gitlab/usage_data.rb | 4 ++-- lib/tasks/import.rake | 2 +- 14 files changed, 22 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index f8645e364ce..282af32ca94 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,6 +1,8 @@ module API module Helpers module Runner + include Gitlab::CurrentSettings + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 10 * 60 diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb index 63f9f8d7a5a..f2bf3d0fb2b 100644 --- a/lib/email_template_interceptor.rb +++ b/lib/email_template_interceptor.rb @@ -1,6 +1,6 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class EmailTemplateInterceptor - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings def self.delivering_email(message) # Remove HTML part if HTML emails are disabled. diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 3d41ac76406..cead1c7eacd 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -6,6 +6,8 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc + extend Gitlab::CurrentSettings + DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8cb4060cd97..1790f380c33 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -19,6 +19,8 @@ module Gitlab OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self + include Gitlab::CurrentSettings + def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7fa02f3d7b3..642f0944354 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,5 +1,7 @@ module Gitlab module CurrentSettings + extend self + def current_application_settings if RequestStore.active? RequestStore.fetch(:current_application_settings) { ensure_application_settings! } diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2d1ae6a5925..9bcc579278f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,6 +3,7 @@ module Gitlab module GonHelper include WebpackHelper + include Gitlab::CurrentSettings def add_gon_variables gon.api_version = 'v4' diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index d7c56463aac..7b06bb953aa 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -1,7 +1,7 @@ module Gitlab module Metrics module InfluxDb - extend Gitlab::CurrentSettings + include Gitlab::CurrentSettings extend self MUTEX = Mutex.new diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 56112ec2301..e73245b82c1 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,6 +1,6 @@ module Gitlab module PerformanceBar - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze EXPIRY_TIME = 5.minutes diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index f0c50584f07..4780675a492 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -1,6 +1,6 @@ module Gitlab class PollingInterval - include Gitlab::CurrentSettings + extend Gitlab::CurrentSettings HEADER_NAME = 'Poll-Interval'.freeze diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index 21aefc884be..09fa14764e6 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -1,5 +1,7 @@ module Gitlab module ProtocolAccess + extend Gitlab::CurrentSettings + def self.allowed?(protocol) if protocol == 'web' true diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index 4bc76ea033f..c463dd487a0 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -1,5 +1,7 @@ module Gitlab module Recaptcha + extend Gitlab::CurrentSettings + def self.load_configurations! if current_application_settings.recaptcha_enabled ::Recaptcha.configure do |config| diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 2442c2ded3b..f6bdd6cf0fe 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -1,5 +1,7 @@ module Gitlab module Sentry + extend Gitlab::CurrentSettings + def self.enabled? Rails.env.production? && current_application_settings.sentry_enabled? end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 748e0a29184..3cf26625108 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,8 +1,8 @@ module Gitlab class UsageData - include Gitlab::CurrentSettings - class << self + include Gitlab::CurrentSettings + def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 1206302cb76..4d485108cf6 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -80,7 +80,7 @@ class GithubImport end def visibility_level - @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility end end -- cgit v1.2.1 From ef030709ebffdecdce67f2693d8c54669edfb7a7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 14:56:25 +0200 Subject: Change kubernetes job policy allowed values It is now possible to use `kubernetes: configured`. --- lib/ci/gitlab_ci_yaml_processor.rb | 5 ++--- lib/gitlab/ci/config/entry/policy.rb | 2 +- lib/gitlab/ci/config/entry/validators.rb | 8 ++++++++ 3 files changed, 11 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 5676618f7fd..434b8948f41 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -94,12 +94,11 @@ module Ci except_kubernetes = job.dig(:except, :kubernetes) [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? end end - def jobs_for_ref(ref, tag = false, source = nil) @jobs.select do |_, job| process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index a8bba3d3ea4..922d568f0ab 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -33,7 +33,7 @@ module Gitlab with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true - validates :kubernetes, inclusion: { in: [true] } + validates :kubernetes, allowed_values: %w[configured] end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b2ca3c881e4..0159179f0a9 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -14,6 +14,14 @@ module Gitlab end end + class AllowedValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless options[:in].include?(value.to_s) + record.errors.add(attribute, "unknown value: #{value}") + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers -- cgit v1.2.1 From 6a2ee0968e811d31fb4cc23b30a6b42e42adf47b Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Thu, 31 Aug 2017 13:44:49 +0200 Subject: API: Use defined project requirements --- lib/api/access_requests.rb | 2 +- lib/api/award_emoji.rb | 2 +- lib/api/boards.rb | 2 +- lib/api/commit_statuses.rb | 2 +- lib/api/deploy_keys.rb | 2 +- lib/api/deployments.rb | 2 +- lib/api/environments.rb | 2 +- lib/api/events.rb | 2 +- lib/api/group_milestones.rb | 2 +- lib/api/group_variables.rb | 2 +- lib/api/groups.rb | 2 +- lib/api/issues.rb | 4 ++-- lib/api/jobs.rb | 2 +- lib/api/labels.rb | 2 +- lib/api/members.rb | 2 +- lib/api/merge_request_diffs.rb | 2 +- lib/api/merge_requests.rb | 2 +- lib/api/notes.rb | 2 +- lib/api/notification_settings.rb | 2 +- lib/api/pipeline_schedules.rb | 2 +- lib/api/pipelines.rb | 2 +- lib/api/project_hooks.rb | 2 +- lib/api/project_milestones.rb | 2 +- lib/api/project_snippets.rb | 2 +- lib/api/projects.rb | 4 ++-- lib/api/repositories.rb | 2 +- lib/api/runners.rb | 2 +- lib/api/services.rb | 4 ++-- lib/api/subscriptions.rb | 2 +- lib/api/todos.rb | 2 +- lib/api/triggers.rb | 2 +- lib/api/variables.rb | 2 +- 32 files changed, 35 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 4fa9b2b2494..374b611f55e 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Gets a list of access requests for a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 8e3851640da..c3d93996816 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do AWARDABLES.each do |awardable_params| awardable_string = awardable_params[:type].pluralize awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}" diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 0d11c5fc971..366b0dc9a6f 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all project boards' do detail 'This feature was introduced in 8.13' success Entities::Board diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 485b680cd5f..78e889a4c35 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -5,7 +5,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include PaginationParams before { authenticate! } diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index f405c341398..281269b1190 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -17,7 +17,7 @@ module API params do requires :id, type: String, desc: 'The ID of the project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authorize_admin_project } desc "Get a specific project's deploy keys" do diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 46b936897f6..1efee9a1324 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all deployments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Deployment diff --git a/lib/api/environments.rb b/lib/api/environments.rb index e33269f9483..5c63ec028d9 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all environments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Environment diff --git a/lib/api/events.rb b/lib/api/events.rb index dabdf579119..b0713ff1d54 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -67,7 +67,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "List a Project's visible events" do success Entities::Event end diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index b85eb59dc0a..93fa0b95857 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of group milestones' do success Entities::Milestone end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 25152f30998..92800ce6450 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -9,7 +9,7 @@ module API requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get group-level variables' do success Entities::Variable end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 8c494a54329..31a918eda60 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -89,7 +89,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Update a group. Available only for users who can administrate groups.' do success Entities::Group end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0297023226f..e4c2c390853 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -81,7 +81,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: { id: %r{[^/]+} } do + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of group issues' do success Entities::IssueBasic end @@ -108,7 +108,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include TimeTrackingEndpoints desc 'Get a list of project issues' do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index a40018b214e..5bab96398fd 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c0cf618ee8d..e41a1720ac1 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all labels of the project' do success Entities::Label end diff --git a/lib/api/members.rb b/lib/api/members.rb index a5d3d7f25a0..22e4bdead41 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Gets a list of group or project members viewable by the authenticated user.' do success Entities::Member end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 4b79eac2b8b..c3affcc6c6b 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of merge request diff versions' do detail 'This feature was introduced in GitLab 8.12.' success Entities::MergeRequestDiff diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index eec8d9357aa..7bcbf9f20ff 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -72,7 +72,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do include TimeTrackingEndpoints helpers do diff --git a/lib/api/notes.rb b/lib/api/notes.rb index e116448c15b..d6e7203adaf 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do NOTEABLE_TYPES.each do |noteable_type| noteables_str = noteable_type.to_s.underscore.pluralize diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index 5d113c94b22..bcc0833aa5c 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -54,7 +54,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Get #{source_type} level notification level settings, defaults to Global" do detail 'This feature was introduced in GitLab 8.12' success Entities::NotificationSetting diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index e3123ef4e2d..ef01cbc7875 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all pipeline schedules' do success Entities::PipelineSchedule end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index e505cae3992..74b3376a1f3 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get all Pipelines of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::PipelineBasic diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 5b457bbe639..86066e2b58f 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -24,7 +24,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get project hooks' do success Entities::ProjectHook end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 451998c726a..0cb209a02d0 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a list of project milestones' do success Entities::Milestone end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 704e8c6718d..2ccda1c1aa1 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 78d900984ac..4845242a173 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -95,7 +95,7 @@ module API end end - resource :users, requirements: { user_id: %r{[^/]+} } do + resource :users, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a user projects' do success Entities::BasicProjectDetails end @@ -183,7 +183,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a single project' do success Entities::ProjectWithAccess end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 14d2bff9cb5..2255fb1b70d 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 68c2120cc15..1ea9a7918d7 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -87,7 +87,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authorize_admin_project } desc 'Get runners available for project' do diff --git a/lib/api/services.rb b/lib/api/services.rb index ff9ddd44439..2cbd0517dc3 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -601,7 +601,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do before { authenticate! } before { authorize_admin_project } @@ -691,7 +691,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc "Trigger a slash command for #{service_slug}" do detail 'Added in GitLab 8.13' end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 91567909998..b3e1e23031a 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -12,7 +12,7 @@ module API requires :id, type: String, desc: 'The ID of a project' requires :subscribable_id, type: String, desc: 'The ID of a resource' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do subscribable_types.each do |type, finder| type_singularized = type.singularize entity_class = Entities.const_get(type_singularized.camelcase) diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 55191169dd4..ffccfebe752 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_iid".to_sym diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index c9fee7e5193..dd6801664b1 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -5,7 +5,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Trigger a GitLab project pipeline' do success Entities::Pipeline end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index da71787abab..d08876ae1b9 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -9,7 +9,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get project variables' do success Entities::Variable end -- cgit v1.2.1 From 8ad690b0d4f9db528c40e7f523e6f8219c944d48 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 23 Aug 2017 17:14:21 +0200 Subject: Prepare GitOperationService for move to Gitlab::Git --- lib/gitlab/conflict/file_collection.rb | 2 +- lib/gitlab/git/repository.rb | 100 +++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index d671867e7c7..90f83e0f810 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -18,7 +18,7 @@ module Gitlab new(merge_request, project).tap do |file_collection| project .repository - .with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + .with_repo_branch_commit(merge_request.target_project.repository.raw_repository, merge_request.target_branch) do yield file_collection end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index fb6504bdea0..7a145107d3d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -731,6 +731,106 @@ module Gitlab end end + def with_repo_branch_commit(start_repository, start_branch_name) + raise "expected Gitlab::Git::Repository, got #{start_repository}" unless start_repository.is_a?(Gitlab::Git::Repository) + + return yield nil if start_repository.empty_repo? + + if start_repository == self + yield commit(start_branch_name) + else + sha = start_repository.commit(start_branch_name).sha + + if branch_commit = commit(sha) + yield branch_commit + else + with_repo_tmp_commit( + start_repository, start_branch_name, sha) do |tmp_commit| + yield tmp_commit + end + end + end + end + + def with_repo_tmp_commit(start_repository, start_branch_name, sha) + tmp_ref = fetch_ref( + start_repository.path, + "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + "refs/tmp/#{SecureRandom.hex}/head" + ) + + yield commit(sha) + ensure + delete_refs(tmp_ref) if tmp_ref + end + + def fetch_source_branch(source_repository, source_branch, local_ref) + with_repo_branch_commit(source_repository, source_branch) do |commit| + if commit + write_ref(local_ref, commit.sha) + else + raise Rugged::ReferenceError, 'source repository is empty' + end + end + end + + def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) + with_repo_branch_commit(source_repository, source_branch_name) do |commit| + break unless commit + + Gitlab::Git::Compare.new( + self, + target_branch_name, + commit.sha, + straight: straight + ) + end + end + + def write_ref(ref_path, sha) + rugged.references.create(ref_path, sha, force: true) + end + + def fetch_ref(source_path, source_ref, target_ref) + args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) + message, status = run_git(args) + + # Make sure ref was created, and raise Rugged::ReferenceError when not + raise Rugged::ReferenceError, message if status != 0 + + target_ref + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def run_git(args) + circuit_breaker.perform do + popen([Gitlab.config.git.bin_path, *args], path) + end + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def commit(ref = 'HEAD') + Gitlab::Git::Commit.find(self, ref) + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb + def empty_repo? + !exists? || !has_visible_content? + end + + # + # Git repository can contains some hidden refs like: + # /refs/notes/* + # /refs/git-as-svn/* + # /refs/pulls/* + # This refs by default not visible in project page and not cloned to client side. + # + # This method return true if repository contains some content visible in project page. + # + def has_visible_content? + branch_count > 0 + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end -- cgit v1.2.1 From ed8f7ed671a5fb6197c9e4759bf13742cf967f50 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 30 Aug 2017 08:05:13 +0200 Subject: Sort templates when fetching them Used to rely on the underlying filesystem to sort the entries, now its forced to be sorted on the name of the template. --- lib/gitlab/template/base_template.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 7ebec8e2cff..7393574ac13 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -18,6 +18,10 @@ module Gitlab { name: name, content: content } end + def <=>(other) + name <=> other.name + end + class << self def all(project = nil) if categories.any? @@ -58,7 +62,7 @@ module Gitlab directory = category_directory(category) files = finder(project).list_files_for(directory) - files.map { |f| new(f, project) } + files.map { |f| new(f, project) }.sort end def category_directory(category) -- cgit v1.2.1 From eaf60bb5441190e2ffcf219b3169bda2237d57cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Tue, 29 Aug 2017 23:10:41 -0300 Subject: Implement /internal/post_receive unified endpoint for PostReceive tasks --- lib/api/helpers/internal_helpers.rb | 4 ++++ lib/api/internal.rb | 17 +++++++++++++- lib/gitlab/reference_counter.rb | 44 +++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/reference_counter.rb (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index ecb79317093..f57ff0f2632 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -42,6 +42,10 @@ module API ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) end + def merge_request_urls + ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) + end + private def set_project diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 8b007869dc3..622bd9650e4 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -68,7 +68,7 @@ module API end get "/merge_request_urls" do - ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) + merge_request_urls end # @@ -155,6 +155,21 @@ module API # render_api_error!(e, 500) # end end + + post '/post_receive' do + status 200 + + PostReceive.perform_async(params[:gl_repository], params[:identifier], + params[:changes]) + broadcast_message = BroadcastMessage.current&.last&.message + reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease + + { + merge_request_urls: merge_request_urls, + broadcast_message: broadcast_message, + reference_counter_decreased: reference_counter_decreased + } + end end end end diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb new file mode 100644 index 00000000000..bb26f1b610a --- /dev/null +++ b/lib/gitlab/reference_counter.rb @@ -0,0 +1,44 @@ +module Gitlab + class ReferenceCounter + REFERENCE_EXPIRE_TIME = 600 + + attr_reader :gl_repository, :key + + def initialize(gl_repository) + @gl_repository = gl_repository + @key = "git-receive-pack-reference-counter:#{gl_repository}" + end + + def value + Gitlab::Redis::SharedState.with { |redis| (redis.get(key) || 0).to_i } + end + + def increase + redis_cmd do |redis| + redis.incr(key) + redis.expire(key, REFERENCE_EXPIRE_TIME) + end + end + + def decrease + redis_cmd do |redis| + current_value = redis.decr(key) + if current_value < 0 + Rails.logger.warn("Reference counter for #{gl_repository} decreased" \ + " when its value was less than 1. Reseting the counter.") + redis.del(key) + end + end + end + + private + + def redis_cmd + Gitlab::Redis::SharedState.with { |redis| yield(redis) } + true + rescue => e + Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") + false + end + end +end -- cgit v1.2.1 From 973c697960b9538222c5a83dfe643107313097cc Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 22 Aug 2017 16:23:43 +0200 Subject: Add spec for languages without plurals --- lib/gitlab/po_linter.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/po_linter.rb b/lib/gitlab/po_linter.rb index 54594949711..721a111e2a0 100644 --- a/lib/gitlab/po_linter.rb +++ b/lib/gitlab/po_linter.rb @@ -66,7 +66,11 @@ module Gitlab def validate_variables(errors, entry) if entry[:msgid_plural].present? validate_variables_in_message(errors, entry[:msgid], entry['msgstr[0]']) - validate_variables_in_message(errors, entry[:msgid_plural], entry['msgstr[1]']) + + # Validate all plurals + entry.keys.select { |key_name| key_name =~ /msgstr\[[1-9]\]/ }.each do |plural_key| + validate_variables_in_message(errors, entry[:msgid_plural], entry[plural_key]) + end else validate_variables_in_message(errors, entry[:msgid], entry[:msgstr]) end @@ -80,6 +84,10 @@ module Gitlab validate_translation(errors, message_id, required_variables) message_translation = join_message(message_translation) + + # We don't need to validate when the message is empty. + # Translations could fallback to the default, or we could be validating a + # language that does not have plurals. unless message_translation.empty? validate_variable_usage(errors, message_translation, required_variables) end -- cgit v1.2.1 From 1da594d39b4b5d6d905ab9a8325d694b3b0fbec7 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 24 Aug 2017 19:11:35 +0200 Subject: Check newlines in translations --- lib/gitlab/po_linter.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/po_linter.rb b/lib/gitlab/po_linter.rb index 721a111e2a0..44abea640c3 100644 --- a/lib/gitlab/po_linter.rb +++ b/lib/gitlab/po_linter.rb @@ -61,6 +61,10 @@ module Gitlab if entry[:msgid].is_a?(Array) errors << "<#{message_id}> is defined over multiple lines, this breaks some tooling." end + + if translations_in_entry(entry).any? { |translation| translation.is_a?(Array) } + errors << "<#{message_id}> has translations defined over multiple lines, this breaks some tooling." + end end def validate_variables(errors, entry) @@ -172,5 +176,17 @@ module Gitlab def join_message(message) Array(message).join end + + def translations_in_entry(entry) + if entry[:msgid_plural].present? + entry.fetch_values(*plural_translation_keys_in_entry(entry)) + else + [entry[:msgstr]] + end + end + + def plural_translation_keys_in_entry(entry) + entry.keys.select { |key| key =~ /msgstr\[\d*\]/ } + end end end -- cgit v1.2.1 From 49b38194775a6f0043a0f7f2d01932fcdea69810 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 24 Aug 2017 19:32:53 +0200 Subject: Only perform `join_message` in `validate_variable_usage` --- lib/gitlab/po_linter.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/po_linter.rb b/lib/gitlab/po_linter.rb index 44abea640c3..162ba4058e6 100644 --- a/lib/gitlab/po_linter.rb +++ b/lib/gitlab/po_linter.rb @@ -86,15 +86,7 @@ module Gitlab validate_unnamed_variables(errors, required_variables) validate_translation(errors, message_id, required_variables) - - message_translation = join_message(message_translation) - - # We don't need to validate when the message is empty. - # Translations could fallback to the default, or we could be validating a - # language that does not have plurals. - unless message_translation.empty? - validate_variable_usage(errors, message_translation, required_variables) - end + validate_variable_usage(errors, message_translation, required_variables) end def validate_translation(errors, message_id, used_variables) @@ -150,6 +142,13 @@ module Gitlab end def validate_variable_usage(errors, translation, required_variables) + translation = join_message(translation) + + # We don't need to validate when the message is empty. + # Translations could fallback to the default, or we could be validating a + # language that does not have plurals. + return if translation.empty? + found_variables = translation.scan(VARIABLE_REGEX) missing_variables = required_variables - found_variables -- cgit v1.2.1 From 0fa0ed7d854761c5f055e421464adb0ff3522411 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 25 Aug 2017 09:04:50 +0200 Subject: Move `PoLinter` into `Gitlab::I18n` --- lib/gitlab/i18n/po_linter.rb | 193 +++++++++++++++++++++++++++++++++++++++++++ lib/gitlab/po_linter.rb | 191 ------------------------------------------ lib/tasks/gettext.rake | 4 +- 3 files changed, 195 insertions(+), 193 deletions(-) create mode 100644 lib/gitlab/i18n/po_linter.rb delete mode 100644 lib/gitlab/po_linter.rb (limited to 'lib') diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb new file mode 100644 index 00000000000..201d73cfe1d --- /dev/null +++ b/lib/gitlab/i18n/po_linter.rb @@ -0,0 +1,193 @@ +require 'simple_po_parser' + +module Gitlab + module I18n + class PoLinter + attr_reader :po_path, :entries, :locale + + VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze + + def initialize(po_path, locale = I18n.locale.to_s) + @po_path = po_path + @locale = locale + end + + def errors + @errors ||= validate_po + end + + def validate_po + if parse_error = parse_po + return 'PO-syntax errors' => [parse_error] + end + + validate_entries + end + + def parse_po + @entries = SimplePoParser.parse(po_path) + nil + rescue SimplePoParser::ParserError => e + @entries = [] + e.message + end + + def validate_entries + errors = {} + + entries.each do |entry| + # Skip validation of metadata + next if entry[:msgid].empty? + + errors_for_entry = validate_entry(entry) + errors[join_message(entry[:msgid])] = errors_for_entry if errors_for_entry.any? + end + + errors + end + + def validate_entry(entry) + errors = [] + + validate_flags(errors, entry) + validate_variables(errors, entry) + validate_newlines(errors, entry) + + errors + end + + def validate_newlines(errors, entry) + message_id = join_message(entry[:msgid]) + + if entry[:msgid].is_a?(Array) + errors << "<#{message_id}> is defined over multiple lines, this breaks some tooling." + end + + if translations_in_entry(entry).any? { |translation| translation.is_a?(Array) } + errors << "<#{message_id}> has translations defined over multiple lines, this breaks some tooling." + end + end + + def validate_variables(errors, entry) + if entry[:msgid_plural].present? + validate_variables_in_message(errors, entry[:msgid], entry['msgstr[0]']) + + # Validate all plurals + entry.keys.select { |key_name| key_name =~ /msgstr\[[1-9]\]/ }.each do |plural_key| + validate_variables_in_message(errors, entry[:msgid_plural], entry[plural_key]) + end + else + validate_variables_in_message(errors, entry[:msgid], entry[:msgstr]) + end + end + + def validate_variables_in_message(errors, message_id, message_translation) + message_id = join_message(message_id) + required_variables = message_id.scan(VARIABLE_REGEX) + + validate_unnamed_variables(errors, required_variables) + validate_translation(errors, message_id, required_variables) + validate_variable_usage(errors, message_translation, required_variables) + end + + def validate_translation(errors, message_id, used_variables) + variables = fill_in_variables(used_variables) + + begin + Gitlab::I18n.with_locale(locale) do + translated = if message_id.include?('|') + FastGettext::Translation.s_(message_id) + else + FastGettext::Translation._(message_id) + end + + translated % variables + end + + # `sprintf` could raise an `ArgumentError` when invalid passing something + # other than a Hash when using named variables + # + # `sprintf` could raise `TypeError` when passing a wrong type when using + # unnamed variables + # + # FastGettext::Translation could raise `RuntimeError` (raised as a string), + # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` + # + # `FastGettext::Translation` could raise `ArgumentError` as subclassess + # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` + rescue ArgumentError, TypeError, RuntimeError => e + errors << "Failure translating to #{locale} with #{variables}: #{e.message}" + end + end + + def fill_in_variables(variables) + if variables.empty? + [] + elsif variables.any? { |variable| unnamed_variable?(variable) } + variables.map do |variable| + variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string + end + else + variables.inject({}) do |hash, variable| + variable_name = variable[/\w+/] + hash[variable_name] = Gitlab::Utils.random_string + hash + end + end + end + + def validate_unnamed_variables(errors, variables) + if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } + errors << 'is combining multiple unnamed variables' + end + end + + def validate_variable_usage(errors, translation, required_variables) + translation = join_message(translation) + + # We don't need to validate when the message is empty. + # Translations could fallback to the default, or we could be validating a + # language that does not have plurals. + return if translation.empty? + + found_variables = translation.scan(VARIABLE_REGEX) + + missing_variables = required_variables - found_variables + if missing_variables.any? + errors << "<#{translation}> is missing: [#{missing_variables.to_sentence}]" + end + + unknown_variables = found_variables - required_variables + if unknown_variables.any? + errors << "<#{translation}> is using unknown variables: [#{unknown_variables.to_sentence}]" + end + end + + def unnamed_variable?(variable_name) + !variable_name.start_with?('%{') + end + + def validate_flags(errors, entry) + if flag = entry[:flag] + errors << "is marked #{flag}" + end + end + + def join_message(message) + Array(message).join + end + + def translations_in_entry(entry) + if entry[:msgid_plural].present? + entry.fetch_values(*plural_translation_keys_in_entry(entry)) + else + [entry[:msgstr]] + end + end + + def plural_translation_keys_in_entry(entry) + entry.keys.select { |key| key =~ /msgstr\[\d*\]/ } + end + end + end +end diff --git a/lib/gitlab/po_linter.rb b/lib/gitlab/po_linter.rb deleted file mode 100644 index 162ba4058e6..00000000000 --- a/lib/gitlab/po_linter.rb +++ /dev/null @@ -1,191 +0,0 @@ -require 'simple_po_parser' - -module Gitlab - class PoLinter - attr_reader :po_path, :entries, :locale - - VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze - - def initialize(po_path, locale = I18n.locale.to_s) - @po_path = po_path - @locale = locale - end - - def errors - @errors ||= validate_po - end - - def validate_po - if parse_error = parse_po - return 'PO-syntax errors' => [parse_error] - end - - validate_entries - end - - def parse_po - @entries = SimplePoParser.parse(po_path) - nil - rescue SimplePoParser::ParserError => e - @entries = [] - e.message - end - - def validate_entries - errors = {} - - entries.each do |entry| - # Skip validation of metadata - next if entry[:msgid].empty? - - errors_for_entry = validate_entry(entry) - errors[join_message(entry[:msgid])] = errors_for_entry if errors_for_entry.any? - end - - errors - end - - def validate_entry(entry) - errors = [] - - validate_flags(errors, entry) - validate_variables(errors, entry) - validate_newlines(errors, entry) - - errors - end - - def validate_newlines(errors, entry) - message_id = join_message(entry[:msgid]) - - if entry[:msgid].is_a?(Array) - errors << "<#{message_id}> is defined over multiple lines, this breaks some tooling." - end - - if translations_in_entry(entry).any? { |translation| translation.is_a?(Array) } - errors << "<#{message_id}> has translations defined over multiple lines, this breaks some tooling." - end - end - - def validate_variables(errors, entry) - if entry[:msgid_plural].present? - validate_variables_in_message(errors, entry[:msgid], entry['msgstr[0]']) - - # Validate all plurals - entry.keys.select { |key_name| key_name =~ /msgstr\[[1-9]\]/ }.each do |plural_key| - validate_variables_in_message(errors, entry[:msgid_plural], entry[plural_key]) - end - else - validate_variables_in_message(errors, entry[:msgid], entry[:msgstr]) - end - end - - def validate_variables_in_message(errors, message_id, message_translation) - message_id = join_message(message_id) - required_variables = message_id.scan(VARIABLE_REGEX) - - validate_unnamed_variables(errors, required_variables) - validate_translation(errors, message_id, required_variables) - validate_variable_usage(errors, message_translation, required_variables) - end - - def validate_translation(errors, message_id, used_variables) - variables = fill_in_variables(used_variables) - - begin - Gitlab::I18n.with_locale(locale) do - translated = if message_id.include?('|') - FastGettext::Translation.s_(message_id) - else - FastGettext::Translation._(message_id) - end - - translated % variables - end - - # `sprintf` could raise an `ArgumentError` when invalid passing something - # other than a Hash when using named variables - # - # `sprintf` could raise `TypeError` when passing a wrong type when using - # unnamed variables - # - # FastGettext::Translation could raise `RuntimeError` (raised as a string), - # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` - # - # `FastGettext::Translation` could raise `ArgumentError` as subclassess - # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` - rescue ArgumentError, TypeError, RuntimeError => e - errors << "Failure translating to #{locale} with #{variables}: #{e.message}" - end - end - - def fill_in_variables(variables) - if variables.empty? - [] - elsif variables.any? { |variable| unnamed_variable?(variable) } - variables.map do |variable| - variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string - end - else - variables.inject({}) do |hash, variable| - variable_name = variable[/\w+/] - hash[variable_name] = Gitlab::Utils.random_string - hash - end - end - end - - def validate_unnamed_variables(errors, variables) - if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } - errors << 'is combining multiple unnamed variables' - end - end - - def validate_variable_usage(errors, translation, required_variables) - translation = join_message(translation) - - # We don't need to validate when the message is empty. - # Translations could fallback to the default, or we could be validating a - # language that does not have plurals. - return if translation.empty? - - found_variables = translation.scan(VARIABLE_REGEX) - - missing_variables = required_variables - found_variables - if missing_variables.any? - errors << "<#{translation}> is missing: [#{missing_variables.to_sentence}]" - end - - unknown_variables = found_variables - required_variables - if unknown_variables.any? - errors << "<#{translation}> is using unknown variables: [#{unknown_variables.to_sentence}]" - end - end - - def unnamed_variable?(variable_name) - !variable_name.start_with?('%{') - end - - def validate_flags(errors, entry) - if flag = entry[:flag] - errors << "is marked #{flag}" - end - end - - def join_message(message) - Array(message).join - end - - def translations_in_entry(entry) - if entry[:msgid_plural].present? - entry.fetch_values(*plural_translation_keys_in_entry(entry)) - else - [entry[:msgstr]] - end - end - - def plural_translation_keys_in_entry(entry) - entry.keys.select { |key| key =~ /msgstr\[\d*\]/ } - end - end -end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index b75da6bf2fc..e1491f29b5e 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -28,11 +28,11 @@ namespace :gettext do linters = files.map do |file| locale = File.basename(File.dirname(file)) - Gitlab::PoLinter.new(file, locale) + Gitlab::I18n::PoLinter.new(file, locale) end pot_file = Rails.root.join('locale/gitlab.pot') - linters.unshift(Gitlab::PoLinter.new(pot_file)) + linters.unshift(Gitlab::I18n::PoLinter.new(pot_file)) failed_linters = linters.select { |linter| linter.errors.any? } -- cgit v1.2.1 From cdaf1072daecd628a89f019b701bc0a2fa27c20e Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 25 Aug 2017 11:23:48 +0200 Subject: Move detailed information of an entry into a separate class --- lib/gitlab/i18n/po_entry.rb | 66 ++++++++++++++++++++++++++++++++++++++++++++ lib/gitlab/i18n/po_linter.rb | 40 ++++++++------------------- 2 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 lib/gitlab/i18n/po_entry.rb (limited to 'lib') diff --git a/lib/gitlab/i18n/po_entry.rb b/lib/gitlab/i18n/po_entry.rb new file mode 100644 index 00000000000..aabb477bbea --- /dev/null +++ b/lib/gitlab/i18n/po_entry.rb @@ -0,0 +1,66 @@ +module Gitlab + module I18n + class PoEntry + attr_reader :entry_data + + def initialize(entry_data) + @entry_data = entry_data + end + + def msgid + entry_data[:msgid] + end + + def metadata? + msgid.empty? + end + + def plural_id + entry_data[:msgid_plural] + end + + def plural? + plural_id.present? + end + + def singular_translation + plural? ? entry_data['msgstr[0]'] : entry_data[:msgstr] + end + + def all_translations + @all_translations ||= entry_data.fetch_values(*translation_keys) + end + + def plural_translations + return [] unless plural? + + # The singular translation is used if there's only translation. This is + # the case for languages without plurals. + return all_translations if all_translations.size == 1 + + entry_data.fetch_values(*plural_translation_keys) + end + + def flag + entry_data[:flag] + end + + private + + def plural_translation_keys + @plural_translation_keys ||= translation_keys.select do |key| + plural_index = key.scan(/\d+/).first.to_i + plural_index > 0 + end + end + + def translation_keys + @translation_keys ||= if plural? + entry_data.keys.select { |key| key =~ /msgstr\[\d+\]/ } + else + [:msgstr] + end + end + end + end +end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 201d73cfe1d..2f6965a19aa 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -25,7 +25,7 @@ module Gitlab end def parse_po - @entries = SimplePoParser.parse(po_path) + @entries = SimplePoParser.parse(po_path).map { |data| Gitlab::I18n::PoEntry.new(data) } nil rescue SimplePoParser::ParserError => e @entries = [] @@ -36,11 +36,10 @@ module Gitlab errors = {} entries.each do |entry| - # Skip validation of metadata - next if entry[:msgid].empty? + next if entry.metadata? errors_for_entry = validate_entry(entry) - errors[join_message(entry[:msgid])] = errors_for_entry if errors_for_entry.any? + errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any? end errors @@ -57,27 +56,24 @@ module Gitlab end def validate_newlines(errors, entry) - message_id = join_message(entry[:msgid]) + message_id = join_message(entry.msgid) - if entry[:msgid].is_a?(Array) + if entry.msgid.is_a?(Array) errors << "<#{message_id}> is defined over multiple lines, this breaks some tooling." end - if translations_in_entry(entry).any? { |translation| translation.is_a?(Array) } + if entry.all_translations.any? { |translation| translation.is_a?(Array) } errors << "<#{message_id}> has translations defined over multiple lines, this breaks some tooling." end end def validate_variables(errors, entry) - if entry[:msgid_plural].present? - validate_variables_in_message(errors, entry[:msgid], entry['msgstr[0]']) + validate_variables_in_message(errors, entry.msgid, entry.singular_translation) - # Validate all plurals - entry.keys.select { |key_name| key_name =~ /msgstr\[[1-9]\]/ }.each do |plural_key| - validate_variables_in_message(errors, entry[:msgid_plural], entry[plural_key]) + if entry.plural? + entry.plural_translations.each do |translation| + validate_variables_in_message(errors, entry.plural_id, translation) end - else - validate_variables_in_message(errors, entry[:msgid], entry[:msgstr]) end end @@ -168,26 +164,12 @@ module Gitlab end def validate_flags(errors, entry) - if flag = entry[:flag] - errors << "is marked #{flag}" - end + errors << "is marked #{entry.flag}" if entry.flag end def join_message(message) Array(message).join end - - def translations_in_entry(entry) - if entry[:msgid_plural].present? - entry.fetch_values(*plural_translation_keys_in_entry(entry)) - else - [entry[:msgstr]] - end - end - - def plural_translation_keys_in_entry(entry) - entry.keys.select { |key| key =~ /msgstr\[\d*\]/ } - end end end end -- cgit v1.2.1 From c6d969949ef98f1b4aebf38ca7f3ed1e59791d48 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 25 Aug 2017 15:23:51 +0200 Subject: Validate the number of plurals in an entry --- lib/gitlab/i18n/po_entry.rb | 33 ++++++++++++++++++++++++++++++++- lib/gitlab/i18n/po_linter.rb | 23 +++++++++++++++++------ 2 files changed, 49 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/i18n/po_entry.rb b/lib/gitlab/i18n/po_entry.rb index aabb477bbea..f2a4bfbd1cd 100644 --- a/lib/gitlab/i18n/po_entry.rb +++ b/lib/gitlab/i18n/po_entry.rb @@ -28,11 +28,16 @@ module Gitlab end def all_translations - @all_translations ||= entry_data.fetch_values(*translation_keys) + @all_translations ||= entry_data.fetch_values(*translation_keys).reject(&:empty?) + end + + def translated? + all_translations.any? end def plural_translations return [] unless plural? + return [] unless translated? # The singular translation is used if there's only translation. This is # the case for languages without plurals. @@ -45,8 +50,34 @@ module Gitlab entry_data[:flag] end + def expected_plurals + return nil unless metadata? + return nil unless plural_information + + nplurals = plural_information['nplurals'].to_i + if nplurals > 0 + nplurals + end + end + + # When a translation is a plural, but only has 1 translation, we could be + # talking about a language in which plural and singular is the same thing. + # In which case we always translate as a plural. + def has_singular? + !plural? || all_translations.size > 1 + end + private + def plural_information + return nil unless metadata? + return @plural_information if defined?(@plural_information) + + if plural_line = entry_data[:msgstr].detect { |metadata_line| metadata_line.starts_with?('Plural-Forms: ') } + @plural_information = Hash[plural_line.scan(/(\w+)=([^;\n]+)/)] + end + end + def plural_translation_keys @plural_translation_keys ||= translation_keys.select do |key| plural_index = key.scan(/\d+/).first.to_i diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 2f6965a19aa..e7c92be1383 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -3,7 +3,7 @@ require 'simple_po_parser' module Gitlab module I18n class PoLinter - attr_reader :po_path, :entries, :locale + attr_reader :po_path, :entries, :metadata, :locale VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze @@ -26,6 +26,7 @@ module Gitlab def parse_po @entries = SimplePoParser.parse(po_path).map { |data| Gitlab::I18n::PoEntry.new(data) } + @metadata = @entries.detect { |entry| entry.metadata? } nil rescue SimplePoParser::ParserError => e @entries = [] @@ -51,24 +52,34 @@ module Gitlab validate_flags(errors, entry) validate_variables(errors, entry) validate_newlines(errors, entry) + validate_number_of_plurals(errors, entry) errors end - def validate_newlines(errors, entry) - message_id = join_message(entry.msgid) + def validate_number_of_plurals(errors, entry) + return unless metadata&.expected_plurals + return unless entry.translated? + + if entry.plural? && entry.all_translations.size != metadata.expected_plurals + errors << "should have #{metadata.expected_plurals} #{'translations'.pluralize(metadata.expected_plurals)}" + end + end + def validate_newlines(errors, entry) if entry.msgid.is_a?(Array) - errors << "<#{message_id}> is defined over multiple lines, this breaks some tooling." + errors << "is defined over multiple lines, this breaks some tooling." end if entry.all_translations.any? { |translation| translation.is_a?(Array) } - errors << "<#{message_id}> has translations defined over multiple lines, this breaks some tooling." + errors << "has translations defined over multiple lines, this breaks some tooling." end end def validate_variables(errors, entry) - validate_variables_in_message(errors, entry.msgid, entry.singular_translation) + if entry.has_singular? + validate_variables_in_message(errors, entry.msgid, entry.singular_translation) + end if entry.plural? entry.plural_translations.each do |translation| -- cgit v1.2.1 From f35a5d0d9919810b14d95808f099a3c652fb17b9 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 30 Aug 2017 10:34:13 +0300 Subject: Split translation & metadata entries into classes --- lib/gitlab/i18n/metadata_entry.rb | 24 +++++++++ lib/gitlab/i18n/po_entry.rb | 94 +++--------------------------------- lib/gitlab/i18n/po_linter.rb | 24 +++++---- lib/gitlab/i18n/translation_entry.rb | 68 ++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 96 deletions(-) create mode 100644 lib/gitlab/i18n/metadata_entry.rb create mode 100644 lib/gitlab/i18n/translation_entry.rb (limited to 'lib') diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb new file mode 100644 index 00000000000..3f9cbae62c8 --- /dev/null +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -0,0 +1,24 @@ +module Gitlab + module I18n + class MetadataEntry < PoEntry + def expected_plurals + return nil unless plural_information + + nplurals = plural_information['nplurals'].to_i + if nplurals > 0 + nplurals + end + end + + private + + def plural_information + return @plural_information if defined?(@plural_information) + + if plural_line = entry_data[:msgstr].detect { |metadata_line| metadata_line.starts_with?('Plural-Forms: ') } + @plural_information = Hash[plural_line.scan(/(\w+)=([^;\n]+)/)] + end + end + end + end +end diff --git a/lib/gitlab/i18n/po_entry.rb b/lib/gitlab/i18n/po_entry.rb index f2a4bfbd1cd..014043cfdd6 100644 --- a/lib/gitlab/i18n/po_entry.rb +++ b/lib/gitlab/i18n/po_entry.rb @@ -1,96 +1,18 @@ module Gitlab module I18n class PoEntry - attr_reader :entry_data - - def initialize(entry_data) - @entry_data = entry_data - end - - def msgid - entry_data[:msgid] - end - - def metadata? - msgid.empty? - end - - def plural_id - entry_data[:msgid_plural] - end - - def plural? - plural_id.present? - end - - def singular_translation - plural? ? entry_data['msgstr[0]'] : entry_data[:msgstr] - end - - def all_translations - @all_translations ||= entry_data.fetch_values(*translation_keys).reject(&:empty?) - end - - def translated? - all_translations.any? - end - - def plural_translations - return [] unless plural? - return [] unless translated? - - # The singular translation is used if there's only translation. This is - # the case for languages without plurals. - return all_translations if all_translations.size == 1 - - entry_data.fetch_values(*plural_translation_keys) - end - - def flag - entry_data[:flag] - end - - def expected_plurals - return nil unless metadata? - return nil unless plural_information - - nplurals = plural_information['nplurals'].to_i - if nplurals > 0 - nplurals + def self.build(entry_data) + if entry_data[:msgid].empty? + MetadataEntry.new(entry_data) + else + TranslationEntry.new(entry_data) end end - # When a translation is a plural, but only has 1 translation, we could be - # talking about a language in which plural and singular is the same thing. - # In which case we always translate as a plural. - def has_singular? - !plural? || all_translations.size > 1 - end - - private - - def plural_information - return nil unless metadata? - return @plural_information if defined?(@plural_information) - - if plural_line = entry_data[:msgstr].detect { |metadata_line| metadata_line.starts_with?('Plural-Forms: ') } - @plural_information = Hash[plural_line.scan(/(\w+)=([^;\n]+)/)] - end - end - - def plural_translation_keys - @plural_translation_keys ||= translation_keys.select do |key| - plural_index = key.scan(/\d+/).first.to_i - plural_index > 0 - end - end + attr_reader :entry_data - def translation_keys - @translation_keys ||= if plural? - entry_data.keys.select { |key| key =~ /msgstr\[\d+\]/ } - else - [:msgstr] - end + def initialize(entry_data) + @entry_data = entry_data end end end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index e7c92be1383..0dc96ac7b9b 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -3,7 +3,7 @@ require 'simple_po_parser' module Gitlab module I18n class PoLinter - attr_reader :po_path, :entries, :metadata, :locale + attr_reader :po_path, :translation_entries, :metadata_entry, :locale VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze @@ -25,20 +25,23 @@ module Gitlab end def parse_po - @entries = SimplePoParser.parse(po_path).map { |data| Gitlab::I18n::PoEntry.new(data) } - @metadata = @entries.detect { |entry| entry.metadata? } + entries = SimplePoParser.parse(po_path).map { |data| Gitlab::I18n::PoEntry.build(data) } + + # The first entry is the metadata entry if there is one. + # This is an entry when empty `msgid` + @metadata_entry = entries.shift if entries.first.is_a?(Gitlab::I18n::MetadataEntry) + @translation_entries = entries + nil rescue SimplePoParser::ParserError => e - @entries = [] + @translation_entries = [] e.message end def validate_entries errors = {} - entries.each do |entry| - next if entry.metadata? - + translation_entries.each do |entry| errors_for_entry = validate_entry(entry) errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any? end @@ -58,11 +61,12 @@ module Gitlab end def validate_number_of_plurals(errors, entry) - return unless metadata&.expected_plurals + return unless metadata_entry&.expected_plurals return unless entry.translated? - if entry.plural? && entry.all_translations.size != metadata.expected_plurals - errors << "should have #{metadata.expected_plurals} #{'translations'.pluralize(metadata.expected_plurals)}" + if entry.plural? && entry.all_translations.size != metadata_entry.expected_plurals + errors << "should have #{metadata_entry.expected_plurals} "\ + "#{'translations'.pluralize(metadata_entry.expected_plurals)}" end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb new file mode 100644 index 00000000000..98095177994 --- /dev/null +++ b/lib/gitlab/i18n/translation_entry.rb @@ -0,0 +1,68 @@ +module Gitlab + module I18n + class TranslationEntry < PoEntry + def msgid + entry_data[:msgid] + end + + def plural_id + entry_data[:msgid_plural] + end + + def plural? + plural_id.present? + end + + def singular_translation + plural? ? entry_data['msgstr[0]'] : entry_data[:msgstr] + end + + def all_translations + @all_translations ||= entry_data.fetch_values(*translation_keys).reject(&:empty?) + end + + def translated? + all_translations.any? + end + + def plural_translations + return [] unless plural? + return [] unless translated? + + # The singular translation is used if there's only translation. This is + # the case for languages without plurals. + return all_translations if all_translations.size == 1 + + entry_data.fetch_values(*plural_translation_keys) + end + + def flag + entry_data[:flag] + end + + # When a translation is a plural, but only has 1 translation, we could be + # talking about a language in which plural and singular is the same thing. + # In which case we always translate as a plural. + def has_singular? + !plural? || all_translations.size > 1 + end + + private + + def plural_translation_keys + @plural_translation_keys ||= translation_keys.select do |key| + plural_index = key.scan(/\d+/).first.to_i + plural_index > 0 + end + end + + def translation_keys + @translation_keys ||= if plural? + entry_data.keys.select { |key| key =~ /msgstr\[\d+\]/ } + else + [:msgstr] + end + end + end + end +end -- cgit v1.2.1 From 2c4f9b7a73cf5de875b2c77880c040e845960a9a Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 30 Aug 2017 10:53:23 +0300 Subject: Check for newlines in different methods on TranslationEntry --- lib/gitlab/i18n/po_linter.rb | 8 ++++++-- lib/gitlab/i18n/translation_entry.rb | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 0dc96ac7b9b..f5ffc6669e4 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -71,11 +71,15 @@ module Gitlab end def validate_newlines(errors, entry) - if entry.msgid.is_a?(Array) + if entry.msgid_contains_newlines? errors << "is defined over multiple lines, this breaks some tooling." end - if entry.all_translations.any? { |translation| translation.is_a?(Array) } + if entry.plural_id_contains_newlines? + errors << "plural is defined over multiple lines, this breaks some tooling." + end + + if entry.translations_contain_newlines? errors << "has translations defined over multiple lines, this breaks some tooling." end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index 98095177994..4fe8f569f9c 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -47,6 +47,18 @@ module Gitlab !plural? || all_translations.size > 1 end + def msgid_contains_newlines? + msgid.is_a?(Array) + end + + def plural_id_contains_newlines? + plural_id.is_a?(Array) + end + + def translations_contain_newlines? + all_translations.any? { |translation| translation.is_a?(Array) } + end + private def plural_translation_keys -- cgit v1.2.1 From abe198723d76cea1b7f151a15789d26a3d22ad4d Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 31 Aug 2017 13:39:41 +0200 Subject: Take `nplurals` into account when validating translations. --- lib/gitlab/i18n/metadata_entry.rb | 13 +++++++---- lib/gitlab/i18n/po_entry.rb | 19 ---------------- lib/gitlab/i18n/po_linter.rb | 23 ++++++++++++------- lib/gitlab/i18n/translation_entry.rb | 44 +++++++++++++++++------------------- 4 files changed, 44 insertions(+), 55 deletions(-) delete mode 100644 lib/gitlab/i18n/po_entry.rb (limited to 'lib') diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb index 3f9cbae62c8..35d57459a3d 100644 --- a/lib/gitlab/i18n/metadata_entry.rb +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -1,13 +1,16 @@ module Gitlab module I18n - class MetadataEntry < PoEntry + class MetadataEntry + attr_reader :entry_data + + def initialize(entry_data) + @entry_data = entry_data + end + def expected_plurals return nil unless plural_information - nplurals = plural_information['nplurals'].to_i - if nplurals > 0 - nplurals - end + plural_information['nplurals'].to_i end private diff --git a/lib/gitlab/i18n/po_entry.rb b/lib/gitlab/i18n/po_entry.rb deleted file mode 100644 index 014043cfdd6..00000000000 --- a/lib/gitlab/i18n/po_entry.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Gitlab - module I18n - class PoEntry - def self.build(entry_data) - if entry_data[:msgid].empty? - MetadataEntry.new(entry_data) - else - TranslationEntry.new(entry_data) - end - end - - attr_reader :entry_data - - def initialize(entry_data) - @entry_data = entry_data - end - end - end -end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index f5ffc6669e4..c3b1fe582af 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -25,12 +25,19 @@ module Gitlab end def parse_po - entries = SimplePoParser.parse(po_path).map { |data| Gitlab::I18n::PoEntry.build(data) } + entries = SimplePoParser.parse(po_path) # The first entry is the metadata entry if there is one. # This is an entry when empty `msgid` - @metadata_entry = entries.shift if entries.first.is_a?(Gitlab::I18n::MetadataEntry) - @translation_entries = entries + if entries.first[:msgid].empty? + @metadata_entry = Gitlab::I18n::MetadataEntry.new(entries.shift) + else + return 'Missing metadata entry.' + end + + @translation_entries = entries.map do |entry_data| + Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals) + end nil rescue SimplePoParser::ParserError => e @@ -64,7 +71,7 @@ module Gitlab return unless metadata_entry&.expected_plurals return unless entry.translated? - if entry.plural? && entry.all_translations.size != metadata_entry.expected_plurals + if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals errors << "should have #{metadata_entry.expected_plurals} "\ "#{'translations'.pluralize(metadata_entry.expected_plurals)}" end @@ -85,11 +92,11 @@ module Gitlab end def validate_variables(errors, entry) - if entry.has_singular? + if entry.has_singular_translation? validate_variables_in_message(errors, entry.msgid, entry.singular_translation) end - if entry.plural? + if entry.has_plural? entry.plural_translations.each do |translation| validate_variables_in_message(errors, entry.plural_id, translation) end @@ -161,8 +168,8 @@ module Gitlab translation = join_message(translation) # We don't need to validate when the message is empty. - # Translations could fallback to the default, or we could be validating a - # language that does not have plurals. + # In this case we fall back to the default, which has all the the + # required variables. return if translation.empty? found_variables = translation.scan(VARIABLE_REGEX) diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index 4fe8f569f9c..75d5aa0cfe1 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -1,6 +1,13 @@ module Gitlab module I18n - class TranslationEntry < PoEntry + class TranslationEntry + attr_reader :nplurals, :entry_data + + def initialize(entry_data, nplurals) + @entry_data = entry_data + @nplurals = nplurals + end + def msgid entry_data[:msgid] end @@ -9,16 +16,17 @@ module Gitlab entry_data[:msgid_plural] end - def plural? + def has_plural? plural_id.present? end def singular_translation - plural? ? entry_data['msgstr[0]'] : entry_data[:msgstr] + all_translations.first if has_singular_translation? end def all_translations - @all_translations ||= entry_data.fetch_values(*translation_keys).reject(&:empty?) + @all_translations ||= entry_data.fetch_values(*translation_keys) + .reject(&:empty?) end def translated? @@ -26,25 +34,22 @@ module Gitlab end def plural_translations - return [] unless plural? + return [] unless has_plural? return [] unless translated? - # The singular translation is used if there's only translation. This is - # the case for languages without plurals. - return all_translations if all_translations.size == 1 - - entry_data.fetch_values(*plural_translation_keys) + @plural_translations ||= if has_singular_translation? + all_translations.drop(1) + else + all_translations + end end def flag entry_data[:flag] end - # When a translation is a plural, but only has 1 translation, we could be - # talking about a language in which plural and singular is the same thing. - # In which case we always translate as a plural. - def has_singular? - !plural? || all_translations.size > 1 + def has_singular_translation? + nplurals > 1 || !has_plural? end def msgid_contains_newlines? @@ -61,15 +66,8 @@ module Gitlab private - def plural_translation_keys - @plural_translation_keys ||= translation_keys.select do |key| - plural_index = key.scan(/\d+/).first.to_i - plural_index > 0 - end - end - def translation_keys - @translation_keys ||= if plural? + @translation_keys ||= if has_plural? entry_data.keys.select { |key| key =~ /msgstr\[\d+\]/ } else [:msgstr] -- cgit v1.2.1 From 538104bdd1f8f8905e2bc514bc5f94d564e3bbef Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 31 Aug 2017 19:00:29 +0200 Subject: Fetch all translation keys using a regex --- lib/gitlab/i18n/translation_entry.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index 75d5aa0cfe1..8d4fec0decd 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -67,11 +67,7 @@ module Gitlab private def translation_keys - @translation_keys ||= if has_plural? - entry_data.keys.select { |key| key =~ /msgstr\[\d+\]/ } - else - [:msgstr] - end + @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ } end end end -- cgit v1.2.1 From 770bcf71bb85c9eff13f4eb14cbd517986da9056 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 30 Aug 2017 20:39:23 +0200 Subject: Form for setting project auto devops settings --- lib/gitlab/import_export/import_export.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 78795dd3d92..d164a9adb2d 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -50,6 +50,7 @@ project_tree: - :push_event_payload - :stages - :statuses + - :project_auto_devops - :triggers - :pipeline_schedules - :services @@ -141,4 +142,4 @@ methods: events: - :action push_event_payload: - - :action \ No newline at end of file + - :action -- cgit v1.2.1 From 35acc4cbbebee9a3e4f876675fcd45bdc3f49d77 Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Fri, 1 Sep 2017 08:45:19 +0000 Subject: Make Gitaly PostUploadPack mandatory --- lib/gitlab/workhorse.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index a362a3a0bc6..e5ad9b5a40c 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -35,10 +35,7 @@ module Gitlab when 'git_receive_pack' Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?( - :post_upload_pack, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT - ) + true when 'info_refs' true else -- cgit v1.2.1 From 254f5ab62b2863c721cf807f1f7dd1e5a183ed67 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 1 Sep 2017 11:40:07 +0200 Subject: Add Gitlab::Git::Repository#== --- lib/gitlab/git/repository.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7a145107d3d..7f80ea80e3e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -70,6 +70,10 @@ module Gitlab delegate :exists?, to: :gitaly_repository_client + def ==(other) + path == other.path + end + # Default branch in the repository def root_ref @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| -- cgit v1.2.1 From 37383d9a9d9706008a7fa1d90079cb019681094b Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 31 Aug 2017 11:21:40 +0100 Subject: Rollsback changes made to signing_enabled. --- lib/gitlab/auth.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8cb4060cd97..e6e06570f13 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -48,10 +48,6 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? - # Nothing to do here if internal auth is disabled and LDAP is - # not configured - return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? - Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) -- cgit v1.2.1 From 73d0cafbe4acf5a14ec196c3cfb38be4e84b67d9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 12:58:22 +0200 Subject: Check if service active when using CI/CD kubernetes policy --- lib/ci/gitlab_ci_yaml_processor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 434b8948f41..d3551907969 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -89,7 +89,7 @@ module Ci builds.select do |build| job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_available? + has_kubernetes = pipeline.has_kubernetes_active? only_kubernetes = job.dig(:only, :kubernetes) except_kubernetes = job.dig(:except, :kubernetes) -- cgit v1.2.1 From fc4fb6e4d212d5cf406a08c96ba8c423477a5408 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 13:03:43 +0200 Subject: Change CI/CD kubernetes policy keyword to `active` --- lib/gitlab/ci/config/entry/policy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 922d568f0ab..0027e9ec8c5 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -33,7 +33,7 @@ module Gitlab with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true - validates :kubernetes, allowed_values: %w[configured] + validates :kubernetes, allowed_values: %w[active] end end end -- cgit v1.2.1 From 4761235f6944d1627346ca835a192c1ed32f745e Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 31 Aug 2017 20:11:22 +0200 Subject: Validate unescaped `%` chars in PO files --- lib/gitlab/i18n/po_linter.rb | 21 ++++++++++++++++++--- lib/gitlab/i18n/translation_entry.rb | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index c3b1fe582af..2e02787a4f4 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -63,10 +63,25 @@ module Gitlab validate_variables(errors, entry) validate_newlines(errors, entry) validate_number_of_plurals(errors, entry) + validate_unescaped_chars(errors, entry) errors end + def validate_unescaped_chars(errors, entry) + if entry.msgid_contains_unescaped_chars? + errors << 'contains unescaped `%`, escape it using `%%`' + end + + if entry.plural_id_contains_unescaped_chars? + errors << 'plural id contains unescaped `%`, escape it using `%%`' + end + + if entry.translations_contain_unescaped_chars? + errors << 'translation contains unescaped `%`, escape it using `%%`' + end + end + def validate_number_of_plurals(errors, entry) return unless metadata_entry&.expected_plurals return unless entry.translated? @@ -79,15 +94,15 @@ module Gitlab def validate_newlines(errors, entry) if entry.msgid_contains_newlines? - errors << "is defined over multiple lines, this breaks some tooling." + errors << 'is defined over multiple lines, this breaks some tooling.' end if entry.plural_id_contains_newlines? - errors << "plural is defined over multiple lines, this breaks some tooling." + errors << 'plural is defined over multiple lines, this breaks some tooling.' end if entry.translations_contain_newlines? - errors << "has translations defined over multiple lines, this breaks some tooling." + errors << 'has translations defined over multiple lines, this breaks some tooling.' end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index 8d4fec0decd..e6c95afca7e 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -1,6 +1,8 @@ module Gitlab module I18n class TranslationEntry + PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/.freeze + attr_reader :nplurals, :entry_data def initialize(entry_data, nplurals) @@ -64,6 +66,22 @@ module Gitlab all_translations.any? { |translation| translation.is_a?(Array) } end + def msgid_contains_unescaped_chars? + contains_unescaped_chars?(msgid) + end + + def plural_id_contains_unescaped_chars? + contains_unescaped_chars?(plural_id) + end + + def translations_contain_unescaped_chars? + all_translations.any? { |translation| contains_unescaped_chars?(translation) } + end + + def contains_unescaped_chars?(string) + string =~ PERCENT_REGEX + end + private def translation_keys -- cgit v1.2.1 From a6ead9d17966dd0fca7c5810947eebf818d524b3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 14:00:41 +0200 Subject: Remove an empty line from YAML processor class --- lib/ci/gitlab_ci_yaml_processor.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index d3551907969..62b44389b15 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,7 +20,6 @@ module Ci raise ValidationError, e.message end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) -- cgit v1.2.1 From 0187018e482f842704aa2cdc2f81375c859ec824 Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Fri, 1 Sep 2017 12:01:50 +0000 Subject: Simplify Git interactions in AfterImportService --- lib/gitlab/git/repository.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 554e40dc8a6..8709f82bcc4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -250,11 +250,17 @@ module Gitlab branch_names + tag_names end + def delete_all_refs_except(prefixes) + delete_refs(*all_ref_names_except(prefixes)) + end + # Returns an Array of all ref names, except when it's matching pattern # # regexp - The pattern for ref names we don't want - def all_ref_names_except(regexp) - rugged.references.reject { |ref| ref.name =~ regexp }.map(&:name) + def all_ref_names_except(prefixes) + rugged.references.reject do |ref| + prefixes.any? { |p| ref.name.start_with?(p) } + end.map(&:name) end # Discovers the default branch based on the repository's available branches -- cgit v1.2.1 From 6cdaa27a537662732cb089bdd3483d76a5a56a9a Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 1 Sep 2017 12:11:59 +0200 Subject: Move GitOperationService to Gitlab::Git --- lib/gitlab/git.rb | 1 + lib/gitlab/git/operation_service.rb | 168 ++++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 lib/gitlab/git/operation_service.rb (limited to 'lib') diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index b6449f27034..8c9acbc9fbe 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -5,6 +5,7 @@ module Gitlab BRANCH_REF_PREFIX = "refs/heads/".freeze CommandError = Class.new(StandardError) + CommitError = Class.new(StandardError) class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb new file mode 100644 index 00000000000..9e6fca8c80c --- /dev/null +++ b/lib/gitlab/git/operation_service.rb @@ -0,0 +1,168 @@ +module Gitlab + module Git + class OperationService + attr_reader :committer, :repository + + def initialize(committer, new_repository) + committer = Gitlab::Git::Committer.from_user(committer) if committer.is_a?(User) + @committer = committer + + # Refactoring aid + unless new_repository.is_a?(Gitlab::Git::Repository) + raise "expected a Gitlab::Git::Repository, got #{new_repository}" + end + + @repository = new_repository + end + + def add_branch(branch_name, newrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + oldrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) + end + + def rm_branch(branch) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name + oldrev = branch.target + newrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) + end + + def add_tag(tag_name, newrev, options = {}) + ref = Gitlab::Git::TAG_REF_PREFIX + tag_name + oldrev = Gitlab::Git::BLANK_SHA + + with_hooks(ref, newrev, oldrev) do |service| + # We want to pass the OID of the tag object to the hooks. For an + # annotated tag we don't know that OID until after the tag object + # (raw_tag) is created in the repository. That is why we have to + # update the value after creating the tag object. Only the + # "post-receive" hook will receive the correct value in this case. + raw_tag = repository.rugged.tags.create(tag_name, newrev, options) + service.newrev = raw_tag.target_id + end + end + + def rm_tag(tag) + ref = Gitlab::Git::TAG_REF_PREFIX + tag.name + oldrev = tag.target + newrev = Gitlab::Git::BLANK_SHA + + update_ref_in_hooks(ref, newrev, oldrev) do + repository.rugged.tags.delete(tag_name) + end + end + + # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, + # it would be created from `start_branch_name`. + # If `start_project` is passed, and the branch doesn't exist, + # it would try to find the commits from it instead of current repository. + def with_branch( + branch_name, + start_branch_name: nil, + start_repository: repository, + &block) + + # Refactoring aid + unless start_repository.is_a?(Gitlab::Git::Repository) + raise "expected a Gitlab::Git::Repository, got #{start_repository}" + end + + start_branch_name = nil if start_repository.empty_repo? + + if start_branch_name && !start_repository.branch_exists?(start_branch_name) + raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.full_path}" + end + + update_branch_with_hooks(branch_name) do + repository.with_repo_branch_commit( + start_repository, + start_branch_name || branch_name, + &block) + end + end + + private + + # Returns [newrev, should_run_after_create, should_run_after_create_branch] + def update_branch_with_hooks(branch_name) + update_autocrlf_option + + was_empty = repository.empty? + + # Make commit + newrev = yield + + unless newrev + raise Gitlab::Git::CommitError.new('Failed to create commit') + end + + branch = repository.find_branch(branch_name) + oldrev = find_oldrev_from_branch(newrev, branch) + + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + + [newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)] + end + + def find_oldrev_from_branch(newrev, branch) + return Gitlab::Git::BLANK_SHA unless branch + + oldrev = branch.target + + if oldrev == repository.rugged.merge_base(newrev, branch.target) + oldrev + else + raise Gitlab::Git::CommitError.new('Branch diverged') + end + end + + def update_ref_in_hooks(ref, newrev, oldrev) + with_hooks(ref, newrev, oldrev) do + update_ref(ref, newrev, oldrev) + end + end + + def with_hooks(ref, newrev, oldrev) + Gitlab::Git::HooksService.new.execute( + committer, + repository, + oldrev, + newrev, + ref) do |service| + + yield(service) + end + end + + # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. + def update_ref(ref, newrev, oldrev) + # We use 'git update-ref' because libgit2/rugged currently does not + # offer 'compare and swap' ref updates. Without compare-and-swap we can + # (and have!) accidentally reset the ref to an earlier state, clobbering + # commits. See also https://github.com/libgit2/libgit2/issues/1534. + command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] + _, status = Gitlab::Popen.popen( + command, + repository.path) do |stdin| + stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") + end + + unless status.zero? + raise Gitlab::Git::CommitError.new( + "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ + " Please refresh and try again.") + end + end + + def update_autocrlf_option + if repository.autocrlf != :input + repository.autocrlf = :input + end + end + end + end +end -- cgit v1.2.1 From e1ef436d1fd6419ce8a08c4ac33bf664786c80b0 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 17 Aug 2017 01:54:05 +0900 Subject: Update application code by the db schema change --- lib/gitlab/ci/stage/seed.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index f81f9347b4d..d9ac46908b0 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -28,10 +28,15 @@ module Gitlab attributes.merge(project: project, ref: pipeline.ref, tag: pipeline.tag, - trigger_request: trigger) + trigger_request: trigger + protected: protected?) end end + def protected? + @protected ||= project.protected_for?(pipeline.ref) + end + def create! pipeline.stages.create!(stage).tap do |stage| builds_attributes = builds.map do |attributes| -- cgit v1.2.1 From eda34b1a1846a5d5b55cc127a32b0c7628580f25 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 21 Aug 2017 17:24:44 +0900 Subject: Add spec. Fix runner setting page. It worked. --- lib/gitlab/ci/stage/seed.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index d9ac46908b0..2bc78b4f004 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -28,7 +28,7 @@ module Gitlab attributes.merge(project: project, ref: pipeline.ref, tag: pipeline.tag, - trigger_request: trigger + trigger_request: trigger, protected: protected?) end end -- cgit v1.2.1 From bbe967abeba7be1db79e34439e74cd113c240b52 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 22 Aug 2017 17:01:11 +0900 Subject: Add the rest of specs --- lib/gitlab/ci/stage/seed.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index 2bc78b4f004..e19aae35a81 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -29,14 +29,10 @@ module Gitlab ref: pipeline.ref, tag: pipeline.tag, trigger_request: trigger, - protected: protected?) + protected: protected_ref?) end end - def protected? - @protected ||= project.protected_for?(pipeline.ref) - end - def create! pipeline.stages.create!(stage).tap do |stage| builds_attributes = builds.map do |attributes| @@ -48,6 +44,12 @@ module Gitlab end end end + + private + + def protected_ref? + @protected_ref ||= project.protected_for?(pipeline.ref) + end end end end -- cgit v1.2.1 From 6f19fc1147a60f279db35428993ac532841195ad Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 24 Aug 2017 01:28:57 +0900 Subject: Add API support --- lib/api/entities.rb | 1 + lib/api/runners.rb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9df9a515990..f13f2d723bb 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -775,6 +775,7 @@ module API expose :tag_list expose :run_untagged expose :locked + expose :access_level expose :version, :revision, :platform, :architecture expose :contacted_at expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? } diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 1ea9a7918d7..d8fc44e5790 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -55,7 +55,8 @@ module API optional :tag_list, type: Array[String], desc: 'The list of tags for a runner' optional :run_untagged, type: Boolean, desc: 'Flag indicating the runner can execute untagged jobs' optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' - at_least_one_of :description, :active, :tag_list, :run_untagged, :locked + optional :access_level, type: Integer, desc: 'The access_level of the runner' + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level end put ':id' do runner = get_runner(params.delete(:id)) -- cgit v1.2.1 From 13b9b5f11a556b2841aabbf46516d1acab79aa0d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 29 Aug 2017 16:09:30 +0900 Subject: Improve API arguments as String --- lib/api/runners.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/runners.rb b/lib/api/runners.rb index d8fc44e5790..d3559ef71be 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -55,7 +55,8 @@ module API optional :tag_list, type: Array[String], desc: 'The list of tags for a runner' optional :run_untagged, type: Boolean, desc: 'Flag indicating the runner can execute untagged jobs' optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' - optional :access_level, type: Integer, desc: 'The access_level of the runner' + optional :access_level, type: String, values: Ci::Runner.access_levels.keys, + desc: 'The access_level of the runner' at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level end put ':id' do -- cgit v1.2.1 From d8478d166b96b7b40b27b162d6afd170a5c8d763 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Sep 2017 17:54:07 +0900 Subject: Fix spec --- lib/api/commit_statuses.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 78e889a4c35..6314ea63197 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -74,7 +74,8 @@ module API source: :external, sha: commit.sha, ref: ref, - user: current_user) + user: current_user, + protected: @project.protected_for?(ref)) end status = GenericCommitStatus.running_or_pending.find_or_initialize_by( @@ -82,7 +83,8 @@ module API pipeline: pipeline, name: name, ref: ref, - user: current_user + user: current_user, + protected: @project.protected_for?(ref) ) optional_attributes = -- cgit v1.2.1 From 706d49b2590cfd70306f14cffdbadd237620a993 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 4 Sep 2017 03:58:54 +0200 Subject: Added dynamic skip reason to SystemCheck --- lib/system_check/base_check.rb | 19 +++++++++++++++++++ lib/system_check/simple_executor.rb | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 7f9e2ffffc2..0f5742dd67f 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -62,6 +62,25 @@ module SystemCheck call_or_return(@skip_reason) || 'skipped' end + # Define a reason why we skipped the SystemCheck (during runtime) + # + # This is used when you need dynamic evaluation like when you have + # multiple reasons why a check can fail + # + # @param [String] reason to be displayed + def skip_reason=(reason) + @skip_reason = reason + end + + # Skip reason defined during runtime + # + # This value have precedence over the one defined in the subclass + # + # @return [String] the reason + def skip_reason + @skip_reason + end + # Does the check support automatically repair routine? # # @return [Boolean] whether check implemented `#repair!` method or not diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 6604b1078cf..00221f77cf4 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -23,7 +23,7 @@ module SystemCheck # # @param [BaseCheck] check class def <<(check) - raise ArgumentError unless check < BaseCheck + raise ArgumentError unless check.is_a?(Class) && check < BaseCheck @checks << check end @@ -48,7 +48,7 @@ module SystemCheck # When implements skip method, we run it first, and if true, skip the check if check.can_skip? && check.skip? - $stdout.puts check_klass.skip_reason.color(:magenta) + $stdout.puts check.skip_reason.try(:color, :magenta) || check_klass.skip_reason.color(:magenta) return end -- cgit v1.2.1 From eb79e5689e15cde3e1f6902feb62953fd49df6ee Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 4 Sep 2017 03:59:37 +0200 Subject: Refactored app/init script check to use dynamic skip reason --- .../app/init_script_up_to_date_check.rb | 28 ++++++++++++---------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb index 015c7ed1731..53a47eb0f42 100644 --- a/lib/system_check/app/init_script_up_to_date_check.rb +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -7,26 +7,22 @@ module SystemCheck set_skip_reason 'skipped (omnibus-gitlab has no init script)' def skip? - omnibus_gitlab? - end + return true if omnibus_gitlab? - def multi_check - recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') + unless init_file_exists? + self.skip_reason = "can't check because of previous errors" - unless File.exist?(SCRIPT_PATH) - $stdout.puts "can't check because of previous errors".color(:magenta) - return + true end + end + + def check? + recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') recipe_content = File.read(recipe_path) script_content = File.read(SCRIPT_PATH) - if recipe_content == script_content - $stdout.puts 'yes'.color(:green) - else - $stdout.puts 'no'.color(:red) - show_error - end + recipe_content == script_content end def show_error @@ -38,6 +34,12 @@ module SystemCheck ) fix_and_rerun end + + private + + def init_file_exists? + File.exist?(SCRIPT_PATH) + end end end end -- cgit v1.2.1 From 71000b394bb7ddf46d3037d62607d159706530f7 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Mon, 4 Sep 2017 04:25:58 +0200 Subject: Refactored Incoming Email checks to use SystemCheck library --- .../incoming_email/foreman_configured_check.rb | 23 ++++ .../incoming_email/imap_authentication_check.rb | 45 ++++++++ .../incoming_email/initd_configured_check.rb | 32 ++++++ .../incoming_email/mail_room_running_check.rb | 43 +++++++ lib/tasks/gitlab/check.rake | 125 ++------------------- 5 files changed, 151 insertions(+), 117 deletions(-) create mode 100644 lib/system_check/incoming_email/foreman_configured_check.rb create mode 100644 lib/system_check/incoming_email/imap_authentication_check.rb create mode 100644 lib/system_check/incoming_email/initd_configured_check.rb create mode 100644 lib/system_check/incoming_email/mail_room_running_check.rb (limited to 'lib') diff --git a/lib/system_check/incoming_email/foreman_configured_check.rb b/lib/system_check/incoming_email/foreman_configured_check.rb new file mode 100644 index 00000000000..1db7bf2b782 --- /dev/null +++ b/lib/system_check/incoming_email/foreman_configured_check.rb @@ -0,0 +1,23 @@ +module SystemCheck + module IncomingEmail + class ForemanConfiguredCheck < SystemCheck::BaseCheck + set_name 'Foreman configured correctly?' + + def check? + path = Rails.root.join('Procfile') + + File.exist?(path) && File.read(path) =~ /^mail_room:/ + end + + def show_error + try_fixing_it( + 'Enable mail_room in your Procfile.' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb new file mode 100644 index 00000000000..dee108d987b --- /dev/null +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -0,0 +1,45 @@ +module SystemCheck + module IncomingEmail + class ImapAuthenticationCheck < SystemCheck::BaseCheck + set_name 'IMAP server credentials are correct?' + + def check? + if mailbox_config + begin + imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) + imap.starttls if config[:start_tls] + imap.login(config[:email], config[:password]) + connected = true + rescue + connected = false + end + end + + connected + end + + def show_error + try_fixing_it( + 'Check that the information in config/gitlab.yml is correct' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + + private + + def mailbox_config + return @config if @config + + config_path = Rails.root.join('config', 'mail_room.yml').to_s + erb = ERB.new(File.read(config_path)) + erb.filename = config_path + config_file = YAML.load(erb.result) + + @config = config_file[:mailboxes]&.first + end + end + end +end diff --git a/lib/system_check/incoming_email/initd_configured_check.rb b/lib/system_check/incoming_email/initd_configured_check.rb new file mode 100644 index 00000000000..ea23b8ef49c --- /dev/null +++ b/lib/system_check/incoming_email/initd_configured_check.rb @@ -0,0 +1,32 @@ +module SystemCheck + module IncomingEmail + class InitdConfiguredCheck < SystemCheck::BaseCheck + set_name 'Init.d configured correctly?' + + def skip? + omnibus_gitlab? + end + + def check? + mail_room_configured? + end + + def show_error + try_fixing_it( + 'Enable mail_room in the init.d configuration.' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + + private + + def mail_room_configured? + path = '/etc/default/gitlab' + File.exist?(path) && File.read(path).include?('mail_room_enabled=true') + end + end + end +end diff --git a/lib/system_check/incoming_email/mail_room_running_check.rb b/lib/system_check/incoming_email/mail_room_running_check.rb new file mode 100644 index 00000000000..c1807501829 --- /dev/null +++ b/lib/system_check/incoming_email/mail_room_running_check.rb @@ -0,0 +1,43 @@ +module SystemCheck + module IncomingEmail + class MailRoomRunningCheck < SystemCheck::BaseCheck + set_name 'MailRoom running?' + + def skip? + return true if omnibus_gitlab? + + unless mail_room_configured? + self.skip_reason = "can't check because of previous errors" + true + end + end + + def check? + mail_room_running? + end + + def show_error + try_fixing_it( + sudo_gitlab('RAILS_ENV=production bin/mail_room start') + ) + for_more_information( + see_installation_guide_section('Install Init Script'), + 'see log/mail_room.log for possible errors' + ) + fix_and_rerun + end + + private + + def mail_room_configured? + path = '/etc/default/gitlab' + File.exist?(path) && File.read(path).include?('mail_room_enabled=true') + end + + def mail_room_running? + ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) + ps_ux.include?("mail_room") + end + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 1bd36bbe20a..ce626d58c66 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -308,133 +308,24 @@ namespace :gitlab do desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab - start_checking "Reply by email" if Gitlab.config.incoming_email.enabled - check_imap_authentication + checks = [ + SystemCheck::IncomingEmail::ImapAuthenticationCheck + ] if Rails.env.production? - check_initd_configured_correctly - check_mail_room_running + checks << SystemCheck::IncomingEmail::InitdConfiguredCheck + checks << SystemCheck::IncomingEmail::MailRoomRunningCheck else - check_foreman_configured_correctly + checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck end - else - puts 'Reply by email is disabled in config/gitlab.yml' - end - - finished_checking "Reply by email" - end - - # Checks - ######################## - - def check_initd_configured_correctly - return if omnibus_gitlab? - - print "Init.d configured correctly? ... " - - path = "/etc/default/gitlab" - - if File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Enable mail_room in the init.d configuration." - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun - end - end - - def check_foreman_configured_correctly - print "Foreman configured correctly? ... " - path = Rails.root.join("Procfile") - - if File.exist?(path) && File.read(path) =~ /^mail_room:/ - puts "yes".color(:green) + SystemCheck.run('Reply by email', checks) else - puts "no".color(:red) - try_fixing_it( - "Enable mail_room in your Procfile." - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun - end - end - - def check_mail_room_running - return if omnibus_gitlab? - - print "MailRoom running? ... " - - path = "/etc/default/gitlab" - - unless File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "can't check because of previous errors".color(:magenta) - return - end - - if mail_room_running? - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("RAILS_ENV=production bin/mail_room start") - ) - for_more_information( - see_installation_guide_section("Install Init Script"), - "see log/mail_room.log for possible errors" - ) - fix_and_rerun - end - end - - def check_imap_authentication - print "IMAP server credentials are correct? ... " - - config_path = Rails.root.join('config', 'mail_room.yml').to_s - erb = ERB.new(File.read(config_path)) - erb.filename = config_path - config_file = YAML.load(erb.result) - - config = config_file[:mailboxes].first - - if config - begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.starttls if config[:start_tls] - imap.login(config[:email], config[:password]) - connected = true - rescue - connected = false - end - end - - if connected - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Check that the information in config/gitlab.yml is correct" - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun + puts 'Reply by email is disabled in config/gitlab.yml' end end - - def mail_room_running? - ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) - ps_ux.include?("mail_room") - end end namespace :ldap do -- cgit v1.2.1 From 7199e882dba1f664ff2d5b889428c086dce7827d Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Sun, 3 Sep 2017 19:32:07 +0200 Subject: Only require `simple_po_parser` in rake task that needs it That way we don't need to install it in production, since it's really not needed there. --- lib/gitlab/i18n/po_linter.rb | 2 -- lib/tasks/gettext.rake | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 2e02787a4f4..7d3ff8c7f58 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -1,5 +1,3 @@ -require 'simple_po_parser' - module Gitlab module I18n class PoLinter diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index e1491f29b5e..f7f2fa2f14c 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -1,4 +1,5 @@ require "gettext_i18n_rails/tasks" +require 'simple_po_parser' namespace :gettext do # Customize list of translatable files -- cgit v1.2.1 From 78dad4cf321eb84aa5decdea34704145adca0c3e Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Mon, 4 Sep 2017 09:27:09 +0200 Subject: Fix tests --- lib/gitlab/import_export/import_export.yml | 2 +- lib/gitlab/import_export/relation_factory.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index d164a9adb2d..0ea1699108b 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -50,7 +50,7 @@ project_tree: - :push_event_payload - :stages - :statuses - - :project_auto_devops + - :auto_devops - :triggers - :pipeline_schedules - :services diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 20580459046..e6e7e3573b0 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,6 +14,7 @@ module Gitlab create_access_levels: 'ProtectedTag::CreateAccessLevel', labels: :project_labels, priorities: :label_priorities, + auto_devops: 'ProjectAutoDevops', label: :project_label }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze -- cgit v1.2.1 From f3f1ad3992c039563188a84787b1e916e367d702 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Sep 2017 14:09:03 +0200 Subject: Add API endpoint for downloading single job artifact --- lib/api/jobs.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'lib') diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 5bab96398fd..41b3b28037c 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -85,6 +85,25 @@ module API present_artifacts!(build.artifacts_file) end + desc 'Download a specific file from artifacts archive' do + detail 'This feature was introduced in GitLab 10.0' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do + authorize_read_builds! + + build = get_build!(params[:job_id]) + not_found! unless build.artifacts? + + entry = build.artifacts_metadata_entry(params[:artifact_path]) + not_found! unless entry.exists? + + Gitlab::Workhorse.send_artifacts_entry(build, entry) + end + desc 'Download the artifacts file from a job' do detail 'This feature was introduced in GitLab 8.10' end -- cgit v1.2.1 From 8ca5c333fd5170a900c7fa28b6bfcbe1a8bc6477 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 18 Aug 2017 17:25:35 +0900 Subject: Extend API: Pipeline Schedule Variable --- lib/api/entities.rb | 3 +- lib/api/pipeline_schedules.rb | 70 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f13f2d723bb..09d7d9ad349 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -819,7 +819,7 @@ module API class Variable < Grape::Entity expose :key, :value - expose :protected?, as: :protected + expose :protected?, as: :protected, if: -> (entity, options) { entity.respond_to?(protected?) } end class Pipeline < PipelineBasic @@ -840,6 +840,7 @@ module API class PipelineScheduleDetails < PipelineSchedule expose :last_pipeline, using: Entities::PipelineBasic + expose :variables, using: Entities::Variable end class EnvironmentBasic < Grape::Entity diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index ef01cbc7875..e82b974c8cd 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -119,6 +119,76 @@ module API destroy_conditionally!(pipeline_schedule) end + + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + resource :variables, requirements: { pipeline_schedule_id: %r{[^/]+} } do + desc 'Create a new pipeline schedule variable' do + success Entities::PipelineScheduleDetails + end + params do + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + end + post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do + authorize! :read_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :update_pipeline_schedule, pipeline_schedule + + variable_params = declared_params(include_missing: false) + variable = pipeline_schedule.variables.create(variable_params) + + if variable.persisted? + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + desc 'Edit a pipeline schedule variable' do + success Entities::PipelineScheduleDetails + end + params do + optional :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + end + put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do + authorize! :read_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :update_pipeline_schedule, pipeline_schedule + + variable = pipeline_schedule.variables.find_by(key: params[:key]) + not_found!('Variable') unless variable + + if variable.update(declared_params(include_missing: false)) + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + desc 'Delete a pipeline schedule variable' do + success Entities::PipelineScheduleDetails + end + params do + requires :key, type: String, desc: 'The key of the variable' + end + delete ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do + authorize! :read_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :admin_pipeline_schedule, pipeline_schedule + + variable = pipeline_schedule.variables.find_by(key: params[:key]) + not_found!('Variable') unless variable + + status :accepted + present variable, with: Entities::Variable + end + end end helpers do -- cgit v1.2.1 From 03f72f0f419b7e05fe9207c90b92b02ef7291cd1 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 22 Aug 2017 02:21:37 +0900 Subject: Add spec (Halfway) --- lib/api/entities.rb | 2 +- lib/api/pipeline_schedules.rb | 103 ++++++++++++++++++++---------------------- 2 files changed, 51 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 09d7d9ad349..81cf7039260 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -819,7 +819,7 @@ module API class Variable < Grape::Entity expose :key, :value - expose :protected?, as: :protected, if: -> (entity, options) { entity.respond_to?(protected?) } + expose :protected?, as: :protected, if: -> (entity, options) { entity.respond_to?(:protected?) } end class Pipeline < PipelineBasic diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index e82b974c8cd..ffb9a09834d 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -120,74 +120,71 @@ module API destroy_conditionally!(pipeline_schedule) end + desc 'Create a new pipeline schedule variable' do + success Entities::Variable + end params do requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' end - resource :variables, requirements: { pipeline_schedule_id: %r{[^/]+} } do - desc 'Create a new pipeline schedule variable' do - success Entities::PipelineScheduleDetails - end - params do - requires :key, type: String, desc: 'The key of the variable' - requires :value, type: String, desc: 'The value of the variable' - end - post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do - authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule - authorize! :update_pipeline_schedule, pipeline_schedule + post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do + authorize! :read_pipeline_schedule, user_project - variable_params = declared_params(include_missing: false) - variable = pipeline_schedule.variables.create(variable_params) + not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :update_pipeline_schedule, pipeline_schedule - if variable.persisted? - present variable, with: Entities::Variable - else - render_validation_error!(variable) - end + variable_params = declared_params(include_missing: false) + variable = pipeline_schedule.variables.create(variable_params) + if variable.persisted? + present variable, with: Entities::Variable + else + render_validation_error!(variable) end + end - desc 'Edit a pipeline schedule variable' do - success Entities::PipelineScheduleDetails - end - params do - optional :key, type: String, desc: 'The key of the variable' - optional :value, type: String, desc: 'The value of the variable' - end - put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do - authorize! :read_pipeline_schedule, user_project + desc 'Edit a pipeline schedule variable' do + success Entities::Variable + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + end + put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do + authorize! :read_pipeline_schedule, user_project - not_found!('PipelineSchedule') unless pipeline_schedule - authorize! :update_pipeline_schedule, pipeline_schedule + not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :update_pipeline_schedule, pipeline_schedule - variable = pipeline_schedule.variables.find_by(key: params[:key]) - not_found!('Variable') unless variable + variable = pipeline_schedule.variables.find_by(key: params[:key]) + not_found!('Variable') unless variable - if variable.update(declared_params(include_missing: false)) - present variable, with: Entities::Variable - else - render_validation_error!(variable) - end + if variable.update(declared_params(include_missing: false)) + present variable, with: Entities::Variable + else + render_validation_error!(variable) end + end - desc 'Delete a pipeline schedule variable' do - success Entities::PipelineScheduleDetails - end - params do - requires :key, type: String, desc: 'The key of the variable' - end - delete ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do - authorize! :read_pipeline_schedule, user_project + desc 'Delete a pipeline schedule variable' do + success Entities::Variable + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :key, type: String, desc: 'The key of the variable' + end + delete ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do + authorize! :read_pipeline_schedule, user_project - not_found!('PipelineSchedule') unless pipeline_schedule - authorize! :admin_pipeline_schedule, pipeline_schedule + not_found!('PipelineSchedule') unless pipeline_schedule + authorize! :admin_pipeline_schedule, pipeline_schedule - variable = pipeline_schedule.variables.find_by(key: params[:key]) - not_found!('Variable') unless variable + variable = pipeline_schedule.variables.find_by(key: params[:key]) + not_found!('Variable') unless variable - status :accepted - present variable, with: Entities::Variable - end + status :accepted + present variable, with: Entities::Variable end end -- cgit v1.2.1 From fb8f32a92cdfe4cca24cb80a91e8fe48d6b0df25 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 22 Aug 2017 21:42:17 +0900 Subject: Finish spec --- lib/api/pipeline_schedules.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index ffb9a09834d..7a3f74006c6 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -184,7 +184,7 @@ module API not_found!('Variable') unless variable status :accepted - present variable, with: Entities::Variable + present variable.destroy, with: Entities::Variable end end -- cgit v1.2.1 From 362f2226a5febb7a3a82e86f4a83e87a870d67b3 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 24 Aug 2017 21:51:46 +0900 Subject: Improve by zj nice catches --- lib/api/entities.rb | 2 +- lib/api/pipeline_schedules.rb | 37 +++++++++++++------------------------ 2 files changed, 14 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 81cf7039260..0092cc14e74 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -819,7 +819,7 @@ module API class Variable < Grape::Entity expose :key, :value - expose :protected?, as: :protected, if: -> (entity, options) { entity.respond_to?(:protected?) } + expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } end class Pipeline < PipelineBasic diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 7a3f74006c6..a6414bfe3f4 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -33,8 +33,6 @@ module API get ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :read_pipeline_schedule, user_project - not_found!('PipelineSchedule') unless pipeline_schedule - present pipeline_schedule, with: Entities::PipelineScheduleDetails end @@ -75,8 +73,6 @@ module API end put ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) @@ -94,8 +90,6 @@ module API end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.own!(current_user) @@ -113,8 +107,6 @@ module API end delete ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :admin_pipeline_schedule, pipeline_schedule destroy_conditionally!(pipeline_schedule) @@ -130,8 +122,6 @@ module API end post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :update_pipeline_schedule, pipeline_schedule variable_params = declared_params(include_missing: false) @@ -153,17 +143,12 @@ module API end put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :update_pipeline_schedule, pipeline_schedule - variable = pipeline_schedule.variables.find_by(key: params[:key]) - not_found!('Variable') unless variable - - if variable.update(declared_params(include_missing: false)) - present variable, with: Entities::Variable + if pipeline_schedule_variable.update(declared_params(include_missing: false)) + present pipeline_schedule_variable, with: Entities::Variable else - render_validation_error!(variable) + render_validation_error!(pipeline_schedule_variable) end end @@ -176,15 +161,10 @@ module API end delete ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do authorize! :read_pipeline_schedule, user_project - - not_found!('PipelineSchedule') unless pipeline_schedule authorize! :admin_pipeline_schedule, pipeline_schedule - variable = pipeline_schedule.variables.find_by(key: params[:key]) - not_found!('Variable') unless variable - status :accepted - present variable.destroy, with: Entities::Variable + present pipeline_schedule_variable.destroy, with: Entities::Variable end end @@ -194,6 +174,15 @@ module API user_project.pipeline_schedules .preload(:owner, :last_pipeline) .find_by(id: params.delete(:pipeline_schedule_id)) + + @pipeline_schedule || not_found!('Pipeline Schedule') + end + + def pipeline_schedule_variable + @pipeline_schedule_variable ||= + pipeline_schedule.variables.find_by(key: params[:key]) + + @pipeline_schedule_variable || not_found!('Pipeline Schedule Variable') end end end -- cgit v1.2.1 From bb22989c388bb7322e95af72c48d8422494d96e7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 30 Aug 2017 20:40:19 +0900 Subject: Improve def pipeline_schedule with authrozation code --- lib/api/pipeline_schedules.rb | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index a6414bfe3f4..51baf12e287 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -31,8 +31,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end get ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project - present pipeline_schedule, with: Entities::PipelineScheduleDetails end @@ -72,7 +70,6 @@ module API optional :active, type: Boolean, desc: 'The activation of pipeline schedule' end put ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.update(declared_params(include_missing: false)) @@ -89,7 +86,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do - authorize! :read_pipeline_schedule, user_project authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule.own!(current_user) @@ -106,7 +102,6 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do - authorize! :read_pipeline_schedule, user_project authorize! :admin_pipeline_schedule, pipeline_schedule destroy_conditionally!(pipeline_schedule) @@ -121,7 +116,6 @@ module API requires :value, type: String, desc: 'The value of the variable' end post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do - authorize! :read_pipeline_schedule, user_project authorize! :update_pipeline_schedule, pipeline_schedule variable_params = declared_params(include_missing: false) @@ -142,7 +136,6 @@ module API optional :value, type: String, desc: 'The value of the variable' end put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do - authorize! :read_pipeline_schedule, user_project authorize! :update_pipeline_schedule, pipeline_schedule if pipeline_schedule_variable.update(declared_params(include_missing: false)) @@ -160,7 +153,6 @@ module API requires :key, type: String, desc: 'The key of the variable' end delete ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do - authorize! :read_pipeline_schedule, user_project authorize! :admin_pipeline_schedule, pipeline_schedule status :accepted @@ -171,18 +163,23 @@ module API helpers do def pipeline_schedule @pipeline_schedule ||= - user_project.pipeline_schedules - .preload(:owner, :last_pipeline) - .find_by(id: params.delete(:pipeline_schedule_id)) - - @pipeline_schedule || not_found!('Pipeline Schedule') + user_project + .pipeline_schedules + .preload(:owner, :last_pipeline) + .find_by(id: params.delete(:pipeline_schedule_id)).tap do |pipeline_schedule| + unless pipeline_schedule || can?(current_user, :read_pipeline_schedule, pipeline_schedule) + not_found!('Pipeline Schedule') + end + end end def pipeline_schedule_variable @pipeline_schedule_variable ||= - pipeline_schedule.variables.find_by(key: params[:key]) - - @pipeline_schedule_variable || not_found!('Pipeline Schedule Variable') + pipeline_schedule.variables.find_by(key: params[:key]).tap do |pipeline_schedule_variable| + unless pipeline_schedule_variable + not_found!('Pipeline Schedule Variable') + end + end end end end -- cgit v1.2.1 From 2f906430fa9efa61b7808e5849611fef6ecb59a5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 4 Sep 2017 21:53:19 +0900 Subject: Fix security breaching --- lib/api/pipeline_schedules.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 51baf12e287..37f32411296 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -167,7 +167,7 @@ module API .pipeline_schedules .preload(:owner, :last_pipeline) .find_by(id: params.delete(:pipeline_schedule_id)).tap do |pipeline_schedule| - unless pipeline_schedule || can?(current_user, :read_pipeline_schedule, pipeline_schedule) + unless can?(current_user, :read_pipeline_schedule, pipeline_schedule) not_found!('Pipeline Schedule') end end -- cgit v1.2.1 From 75130a41ba19b80ac7b2300721915787ac4681bf Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 24 Aug 2017 17:08:32 +0900 Subject: Remove CreateTriggerRequestService and forbit to save variables on Ci::TriggerRequest --- lib/api/triggers.rb | 8 ++++---- lib/api/v3/triggers.rb | 31 +++++++++++++++++++------------ 2 files changed, 23 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index dd6801664b1..276d3b5fb79 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -15,16 +15,16 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do + authenticate! + authorize! :admin_build, user_project + # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end - project = find_project(params[:id]) - not_found! unless project - - result = Ci::PipelineTriggerService.new(project, nil, params).execute + result = Ci::PipelineTriggerService.new(user_project, nil, params).execute not_found! unless result if result[:http_status] diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index e9d4c35307b..ffc1a38acbc 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -16,25 +16,32 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do - project = find_project(params[:id]) - trigger = Ci::Trigger.find_by_token(params[:token].to_s) - not_found! unless project && trigger - unauthorized! unless trigger.project == project + authenticate! + authorize! :admin_build, user_project # validate variables - variables = params[:variables].to_h - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + params[:variables] = params[:variables].to_h + unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end - # create request and trigger builds - result = Ci::CreateTriggerRequestService.execute(project, trigger, params[:ref].to_s, variables) - pipeline = result.pipeline + result = Ci::PipelineTriggerService.new(user_project, nil, params).execute + not_found! unless result - if pipeline.persisted? - present result.trigger_request, with: ::API::V3::Entities::TriggerRequest + if result[:http_status] + render_api_error!(result[:message], result[:http_status]) else - render_validation_error!(pipeline) + pipeline = result[:pipeline] + trigger_request = pipeline.trigger_request + + # Ws swtiched to Ci::PipelineVariable from Ci::TriggerRequest.variables. + # Ci::TriggerRequest doesn't save variables anymore. + # Although, to prevent braking compatibility, copying variables and present it as Ci::TriggerRequest. + pipeline.variables.each do |variable| + trigger_request.variables << { key: variable.key, value: variable.value } + end + + present trigger_request, with: ::API::V3::Entities::TriggerRequest end end -- cgit v1.2.1 From acc7497855167d4f6ba481422112645a1a04a885 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 26 Aug 2017 04:04:57 +0900 Subject: Revert autheticate! in Trigger API --- lib/api/triggers.rb | 8 ++++---- lib/api/v3/triggers.rb | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 276d3b5fb79..dd6801664b1 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -15,16 +15,16 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do - authenticate! - authorize! :admin_build, user_project - # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end - result = Ci::PipelineTriggerService.new(user_project, nil, params).execute + project = find_project(params[:id]) + not_found! unless project + + result = Ci::PipelineTriggerService.new(project, nil, params).execute not_found! unless result if result[:http_status] diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index ffc1a38acbc..e1da96104a5 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -16,16 +16,16 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do - authenticate! - authorize! :admin_build, user_project - # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end - result = Ci::PipelineTriggerService.new(user_project, nil, params).execute + project = find_project(params[:id]) + not_found! unless project + + result = Ci::PipelineTriggerService.new(project, nil, params).execute not_found! unless result if result[:http_status] -- cgit v1.2.1 From cff104ec4b0dd2c53ed907ab7ca423b7c587dee8 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 28 Aug 2017 23:29:28 +0900 Subject: Fix spec --- lib/api/v3/triggers.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index e1da96104a5..497ccbe20c6 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -34,9 +34,10 @@ module API pipeline = result[:pipeline] trigger_request = pipeline.trigger_request - # Ws swtiched to Ci::PipelineVariable from Ci::TriggerRequest.variables. + # We switched to Ci::PipelineVariable from Ci::TriggerRequest.variables. # Ci::TriggerRequest doesn't save variables anymore. - # Although, to prevent braking compatibility, copying variables and present it as Ci::TriggerRequest. + # Here is copying Ci::PipelineVariable to Ci::TriggerRequest.variables for presenting the variables. + # The same endpoint in v4 API pressents Pipeline instead of TriggerRequest, so it doesn't need such a process. pipeline.variables.each do |variable| trigger_request.variables << { key: variable.key, value: variable.value } end -- cgit v1.2.1 From d614c431055286eaab3b82e810186ac19a2c4fd7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Sep 2017 00:17:56 +0900 Subject: Fix trigger_request.variables --- lib/api/v3/triggers.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index 497ccbe20c6..13cfb9fe38b 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -32,15 +32,13 @@ module API render_api_error!(result[:message], result[:http_status]) else pipeline = result[:pipeline] - trigger_request = pipeline.trigger_request + trigger_request = Ci::TriggerRequest.find_by(commit_id: pipeline.id) # We switched to Ci::PipelineVariable from Ci::TriggerRequest.variables. # Ci::TriggerRequest doesn't save variables anymore. # Here is copying Ci::PipelineVariable to Ci::TriggerRequest.variables for presenting the variables. # The same endpoint in v4 API pressents Pipeline instead of TriggerRequest, so it doesn't need such a process. - pipeline.variables.each do |variable| - trigger_request.variables << { key: variable.key, value: variable.value } - end + trigger_request.variables = params[:variables] present trigger_request, with: ::API::V3::Entities::TriggerRequest end -- cgit v1.2.1 From 5b88bd81f52b454c80c54bef9951bdb8cf607238 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 4 Sep 2017 22:21:47 +0900 Subject: Move trigger_variables to presenter --- lib/gitlab/gitaly_client.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 9a5f4f598b2..72065811528 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -23,7 +23,8 @@ module Gitlab klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) addr = address(storage) addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' - klass.new(addr, :this_channel_is_insecure) + timeout = 10 if Rails.env.test? + klass.new(addr, :this_channel_is_insecure, timeout: timeout) end end end -- cgit v1.2.1 From 209549c95fb0d4ba3196cc8ed888f7c024216ebb Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 4 Sep 2017 22:38:18 +0900 Subject: Revert iunnecessary change(gitaly) --- lib/gitlab/gitaly_client.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 72065811528..9a5f4f598b2 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -23,8 +23,7 @@ module Gitlab klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) addr = address(storage) addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' - timeout = 10 if Rails.env.test? - klass.new(addr, :this_channel_is_insecure, timeout: timeout) + klass.new(addr, :this_channel_is_insecure) end end end -- cgit v1.2.1 From bcd70c4c46ae71366580c7352ddb28075cdf0e60 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Mon, 4 Sep 2017 15:44:46 +0200 Subject: Incorporate review --- lib/gitlab/import_export/relation_factory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index e6e7e3573b0..7448b806b72 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,7 +14,7 @@ module Gitlab create_access_levels: 'ProtectedTag::CreateAccessLevel', labels: :project_labels, priorities: :label_priorities, - auto_devops: 'ProjectAutoDevops', + auto_devops: :project_auto_devops, label: :project_label }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze -- cgit v1.2.1 From 48f017d1e84498eec38d276d94918021a985bfee Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 5 Sep 2017 01:22:57 +0900 Subject: Use pipeline.trigger_requests.last --- lib/api/v3/triggers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index 13cfb9fe38b..534911fde5c 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -32,12 +32,12 @@ module API render_api_error!(result[:message], result[:http_status]) else pipeline = result[:pipeline] - trigger_request = Ci::TriggerRequest.find_by(commit_id: pipeline.id) # We switched to Ci::PipelineVariable from Ci::TriggerRequest.variables. # Ci::TriggerRequest doesn't save variables anymore. # Here is copying Ci::PipelineVariable to Ci::TriggerRequest.variables for presenting the variables. # The same endpoint in v4 API pressents Pipeline instead of TriggerRequest, so it doesn't need such a process. + trigger_request = pipeline.trigger_requests.last trigger_request.variables = params[:variables] present trigger_request, with: ::API::V3::Entities::TriggerRequest -- cgit v1.2.1 From 48115be509ce00120d0609f5f18a5bc3804bb21f Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Wed, 30 Aug 2017 12:00:39 +0100 Subject: Add a system check for the git user's custom SSH configuration --- .../app/git_user_default_ssh_config_check.rb | 69 ++++++++++++++++++++++ lib/tasks/gitlab/check.rake | 1 + 2 files changed, 70 insertions(+) create mode 100644 lib/system_check/app/git_user_default_ssh_config_check.rb (limited to 'lib') diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb new file mode 100644 index 00000000000..7b486d78cf0 --- /dev/null +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -0,0 +1,69 @@ +module SystemCheck + module App + class GitUserDefaultSSHConfigCheck < SystemCheck::BaseCheck + # These files are allowed in the .ssh directory. The `config` file is not + # whitelisted as it may change the SSH client's behaviour dramatically. + WHITELIST = %w[ + authorized_keys + authorized_keys2 + known_hosts + ].freeze + + set_name 'Git user has default SSH configuration?' + set_skip_reason 'skipped (git user is not present or configured)' + + def skip? + !home_dir || !File.directory?(home_dir) + end + + def check? + forbidden_files.empty? + end + + def show_error + backup_dir = "~/gitlab-check-backup-#{Time.now.to_i}" + + instructions = forbidden_files.map do |filename| + "sudo mv #{Shellwords.escape(filename)} #{backup_dir}" + end + + try_fixing_it("mkdir #{backup_dir}", *instructions) + for_more_information('doc/ssh/README.md in section "SSH on the GitLab server"') + fix_and_rerun + end + + private + + def git_user + Gitlab.config.gitlab.user + end + + def home_dir + return @home_dir if defined?(@home_dir) + + @home_dir = + begin + File.expand_path("~#{git_user}") + rescue ArgumentError + nil + end + end + + def ssh_dir + return nil unless home_dir + + File.join(home_dir, '.ssh') + end + + def forbidden_files + @forbidden_files ||= + begin + present = Dir[File.join(ssh_dir, '*')] + whitelisted = WHITELIST.map { |basename| File.join(ssh_dir, basename) } + + present - whitelisted + end + end + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 1bd36bbe20a..92a3f503fcb 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -33,6 +33,7 @@ namespace :gitlab do SystemCheck::App::RedisVersionCheck, SystemCheck::App::RubyVersionCheck, SystemCheck::App::GitVersionCheck, + SystemCheck::App::GitUserDefaultSSHConfigCheck, SystemCheck::App::ActiveUsersCheck ] -- cgit v1.2.1 From f5bb1789ee170cbdfb7580f6ec5de17ce06b6cbc Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Mon, 4 Sep 2017 16:55:29 -0300 Subject: Fix specs BACKPORT --- lib/gitlab/path_regex.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 894bd5efae5..7c02c9c5c48 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -26,6 +26,7 @@ module Gitlab apple-touch-icon.png assets autocomplete + boards ci dashboard deploy.html -- cgit v1.2.1 From cf9c54bd312dcb7b4b7f0602e83013d8d32a9413 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 3 Sep 2017 16:34:50 +0900 Subject: Add my_reaction_emoji param to /issues API --- lib/api/issues.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/issues.rb b/lib/api/issues.rb index e4c2c390853..1729df2aad0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -36,6 +36,7 @@ module API optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination end -- cgit v1.2.1 From 7e42711659d514be77119698d2611830fb83090d Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 4 Sep 2017 10:43:14 +0900 Subject: Add my_reaction_emoji param to /merge_requests API --- lib/api/merge_requests.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 7bcbf9f20ff..56d72d511da 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -40,6 +40,7 @@ module API optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' + optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination end end -- cgit v1.2.1 From dcf09d11447c264f4b4028ea80eea2be913c2f5b Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 31 Aug 2017 03:20:54 +0900 Subject: Implement `failure_reason` on `ci_builds` --- lib/api/commit_statuses.rb | 2 +- lib/api/runner.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 6314ea63197..c129cc9171d 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -103,7 +103,7 @@ module API when 'success' status.success! when 'failed' - status.drop! + status.drop!(:api) when 'canceled' status.cancel! else diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 11999354594..604bfd53296 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -127,7 +127,7 @@ module API when 'success' job.success when 'failed' - job.drop + job.drop(:failed_job_state) end end -- cgit v1.2.1 From b1af1f268b97c8518bf2806bca48f49174a8aead Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 31 Aug 2017 03:36:15 +0900 Subject: Fix enum wording --- lib/api/commit_statuses.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index c129cc9171d..9ab64452d2b 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -103,7 +103,7 @@ module API when 'success' status.success! when 'failed' - status.drop!(:api) + status.drop!(:failed_by_api) when 'canceled' status.cancel! else -- cgit v1.2.1 From 1d7c0390722c96aa66af5b26f5a826b97293dcd6 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 31 Aug 2017 22:03:41 +0900 Subject: Fix enum lists --- lib/api/commit_statuses.rb | 2 +- lib/api/runner.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 9ab64452d2b..829eef18795 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -103,7 +103,7 @@ module API when 'success' status.success! when 'failed' - status.drop!(:failed_by_api) + status.drop!(:api_failure) when 'canceled' status.cancel! else diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 604bfd53296..701c1bff1e0 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -127,7 +127,7 @@ module API when 'success' job.success when 'failed' - job.drop(:failed_job_state) + job.drop(:job_failure) end end -- cgit v1.2.1 From 68f6c61cf621db82ac98d561782590b1866fcf6f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Sep 2017 16:52:11 +0900 Subject: - Allow runner API to pass failure_reason - Fix spec --- lib/api/runner.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 701c1bff1e0..44ca42fef68 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -114,6 +114,8 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :trace, type: String, desc: %q(Job's full trace) optional :state, type: String, desc: %q(Job's status: success, failed) + optional :failure_reason, type: String, values: CommitStatus.failure_reasons.keys, + desc: %q(Job's failure_reason) end put '/:id' do job = authenticate_job! @@ -127,7 +129,11 @@ module API when 'success' job.success when 'failed' - job.drop(:job_failure) + if params[:failure_reason] + job.drop(params[:failure_reason].to_sym) + else + job.drop(:job_failure) + end end end -- cgit v1.2.1 From 5d50cbfaab1d67ffaea6064aeac848f1fc1127a6 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 4 Sep 2017 22:46:37 +0900 Subject: Use unknown_failure for runner --- lib/api/runner.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 44ca42fef68..76354fd8c98 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -129,11 +129,8 @@ module API when 'success' job.success when 'failed' - if params[:failure_reason] - job.drop(params[:failure_reason].to_sym) - else - job.drop(:job_failure) - end + failure_reason = params[:failure_reason] ? params[:failure_reason].to_sym : :unknown_failure + job.drop(failure_reason) end end -- cgit v1.2.1 From 38d9b4d77d85e26f827ff9640243494adc8597ed Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 5 Sep 2017 15:10:34 +0900 Subject: Use script_failure. Add runner_system_failure. Improve spec. --- lib/api/runner.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 76354fd8c98..a3987c560dd 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -129,8 +129,7 @@ module API when 'success' job.success when 'failed' - failure_reason = params[:failure_reason] ? params[:failure_reason].to_sym : :unknown_failure - job.drop(failure_reason) + job.drop(params[:failure_reason] || :unknown_failure) end end -- cgit v1.2.1 From dfb8fcbb651812d209d2f42baf6c2bb0e851c861 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 5 Sep 2017 11:16:49 +0200 Subject: Use API helper to send artifact file through Workhorse --- lib/api/helpers.rb | 4 ++++ lib/api/jobs.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3d377fdb9eb..f9ce1165544 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -432,6 +432,10 @@ module API header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) end + def send_artifacts_entry(build, entry) + header(*Gitlab::Workhorse.send_artifacts_entry(build, entry)) + end + # The Grape Error Middleware only has access to env but no params. We workaround this by # defining a method that returns the right value. def define_params_for_grape_middleware diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 41b3b28037c..41c70a2dcb7 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -101,7 +101,7 @@ module API entry = build.artifacts_metadata_entry(params[:artifact_path]) not_found! unless entry.exists? - Gitlab::Workhorse.send_artifacts_entry(build, entry) + send_artifacts_entry(build, entry) end desc 'Download the artifacts file from a job' do -- cgit v1.2.1 From 42062a454a650d81d9fe6dddde7b39b056ec0a88 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 24 Aug 2017 18:17:04 +0200 Subject: Re-use issue/MR counts for the pagination system This changes the issue and MR index pages so the pagination system re-uses the output of the COUNT(*) query used to calculate the number of rows per state (opened, closed, etc). This removes the need for an additional COUNT(*) on both pages. --- lib/gitlab/issuables_count_for_state.rb | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 lib/gitlab/issuables_count_for_state.rb (limited to 'lib') diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb new file mode 100644 index 00000000000..505810964bc --- /dev/null +++ b/lib/gitlab/issuables_count_for_state.rb @@ -0,0 +1,50 @@ +module Gitlab + # Class for counting and caching the number of issuables per state. + class IssuablesCountForState + # The name of the RequestStore cache key. + CACHE_KEY = :issuables_count_for_state + + # The state values that can be safely casted to a Symbol. + STATES = %w[opened closed merged all].freeze + + # finder - The finder class to use for retrieving the issuables. + def initialize(finder) + @finder = finder + @cache = + if RequestStore.active? + RequestStore[CACHE_KEY] ||= initialize_cache + else + initialize_cache + end + end + + def for_state_or_opened(state = nil) + self[state || :opened] + end + + # Returns the count for the given state. + # + # state - The name of the state as either a String or a Symbol. + # + # Returns an Integer. + def [](state) + state = state.to_sym if cast_state_to_symbol?(state) + + cache_for_finder[state] || 0 + end + + private + + def cache_for_finder + @cache[@finder] + end + + def cast_state_to_symbol?(state) + state.is_a?(String) && STATES.include?(state) + end + + def initialize_cache + Hash.new { |hash, finder| hash[finder] = finder.count_by_state } + end + end +end -- cgit v1.2.1 From 02aa269e34b7c05a52e5226fc02d91ffbbb6010f Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Tue, 5 Sep 2017 10:55:12 +1100 Subject: Add branch existence check to the APIv4 branches via HEAD request --- lib/api/branches.rb | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/api/branches.rb b/lib/api/branches.rb index a989394ad91..642c1140fcc 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -24,17 +24,22 @@ module API present paginate(branches), with: Entities::RepoBranch, project: user_project end - desc 'Get a single branch' do - success Entities::RepoBranch - end - params do - requires :branch, type: String, desc: 'The name of the branch' - end - get ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do - branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless branch + resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + desc 'Get a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, desc: 'The name of the branch' + end + head do + user_project.repository.branch_exists?(params[:branch]) ? status(204) : status(404) + end + get do + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project + end end # Note: This API will be deprecated in favor of the protected branches API. -- cgit v1.2.1 From 9d88ad45d2d60b390e697097f0184d79aac4a8eb Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 1 Sep 2017 16:16:42 +0200 Subject: Add Gitaly migration points for branch/tag create/delete --- lib/gitlab/git/repository.rb | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index c67f7724307..f7577b02d5d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -605,6 +605,49 @@ module Gitlab # TODO: implement this method end + def add_branch(branch_name, committer:, target:) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + OperationService.new(committer, self).add_branch(branch_name, target_object.oid) + find_branch(branch_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def add_tag(tag_name, committer:, target:, message: nil) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + committer = Committer.from_user(committer) if committer.is_a?(User) + + options = nil # Use nil, not the empty hash. Rugged cares about this. + if message + options = { + message: message, + tagger: Gitlab::Git.committer_hash(email: committer.email, name: committer.name) + } + end + + OperationService.new(committer, self).add_tag(tag_name, target_object.oid, options) + + find_tag(tag_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def rm_branch(branch_name, committer:) + OperationService.new(committer, self).rm_branch(find_branch(branch_name)) + end + + def rm_tag(tag_name, committer:) + OperationService.new(committer, self).rm_tag(find_tag(tag_name)) + end + + def find_tag(name) + tags.find { |tag| tag.name == name } + end + # Delete the specified branch from the repository # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 -- cgit v1.2.1 From c53f319f883064e6c5e77f9b48a235b23b0f4363 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 5 Sep 2017 12:17:56 +0200 Subject: Extract a class that represents artifacts file path --- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 244 ++++++++++++------------ lib/gitlab/ci/build/artifacts/path.rb | 51 +++++ 2 files changed, 173 insertions(+), 122 deletions(-) create mode 100644 lib/gitlab/ci/build/artifacts/path.rb (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 2e073334abc..321ddfd8c29 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -1,129 +1,129 @@ module Gitlab - module Ci::Build::Artifacts - class Metadata - ## - # Class that represents an entry (path and metadata) to a file or - # directory in GitLab CI Build Artifacts binary file / archive - # - # This is IO-operations safe class, that does similar job to - # Ruby's Pathname but without the risk of accessing filesystem. - # - # This class is working only with UTF-8 encoded paths. - # - class Entry - attr_reader :path, :entries - attr_accessor :name - - def initialize(path, entries) - @path = path.dup.force_encoding('UTF-8') - @entries = entries - - if path.include?("\0") - raise ArgumentError, 'Path contains zero byte character!' - end + module Ci + module Build + module Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :entries + attr_accessor :name + + def initialize(path, entries) + @entries = entries + @path = Artifacts::Path.new(path) + end + + delegate :empty?, to: :children + + def directory? + blank_node? || @path.directory? + end + + def file? + !directory? + end + + def blob + return unless file? + + @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.to_s.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.name + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path.to_s)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path.to_s] || {} + end + + def nodes + @path.nodes + (file? ? 1 : 0) + end + + def blank_node? + @path.to_s.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path.to_s) + end + + def total_size + descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}} + entries.sum do |path, entry| + (entry[:size] if path =~ descendant_pattern).to_i + end + end + + def path + @path.to_s + end + + def to_s + @path.to_s + end + + def ==(other) + path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{self.to_s}" + end - unless path.valid_encoding? - raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end end end - - delegate :empty?, to: :children - - def directory? - blank_node? || @path.end_with?('/') - end - - def file? - !directory? - end - - def blob - return unless file? - - @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) - end - - def has_parent? - nodes > 0 - end - - def parent - return nil unless has_parent? - self.class.new(@path.chomp(basename), @entries) - end - - def basename - (directory? && !blank_node?) ? name + '/' : name - end - - def name - @name || @path.split('/').last.to_s - end - - def children - return [] unless directory? - return @children if @children - - child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select_entries { |path| path =~ child_pattern } - end - - def directories(opts = {}) - return [] unless directory? - dirs = children.select(&:directory?) - return dirs unless has_parent? && opts[:parent] - - dotted_parent = parent - dotted_parent.name = '..' - dirs.prepend(dotted_parent) - end - - def files - return [] unless directory? - children.select(&:file?) - end - - def metadata - @entries[@path] || {} - end - - def nodes - @path.count('/') + (file? ? 1 : 0) - end - - def blank_node? - @path.empty? # "" is considered to be './' - end - - def exists? - blank_node? || @entries.include?(@path) - end - - def total_size - descendant_pattern = %r{^#{Regexp.escape(@path)}} - entries.sum do |path, entry| - (entry[:size] if path =~ descendant_pattern).to_i - end - end - - def to_s - @path - end - - def ==(other) - @path == other.path && @entries == other.entries - end - - def inspect - "#{self.class.name}: #{@path}" - end - - private - - def select_entries - selected = @entries.select { |path, _metadata| yield path } - selected.map { |path, _metadata| self.class.new(path, @entries) } - end end end end diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb new file mode 100644 index 00000000000..9cd9b36c5f8 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/path.rb @@ -0,0 +1,51 @@ +module Gitlab + module Ci + module Build + module Artifacts + class Path + def initialize(path) + @path = path.dup.force_encoding('UTF-8') + end + + def valid? + nonzero? && utf8? + end + + def directory? + @path.end_with?('/') + end + + def name + @path.split('/').last.to_s + end + + def nodes + @path.count('/') + end + + def to_s + @path.tap do |path| + unless nonzero? + raise ArgumentError, 'Path contains zero byte character!' + end + + unless utf8? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + end + + private + + def nonzero? + @path.exclude?("\0") + end + + def utf8? + @path.valid_encoding? + end + end + end + end + end +end -- cgit v1.2.1 From 508ff17b3405a4e2275fa137bd7322b728db8ed4 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 24 Aug 2017 14:21:26 +0200 Subject: pass whole commit to Gitlab::Gpg::Commit again we need the commit object for the updated verification that also checks the committer's email to match the gpg key and user's emails. --- lib/gitlab/gpg/commit.rb | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 606c7576f70..f701897955b 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -1,17 +1,12 @@ module Gitlab module Gpg class Commit - def self.for_commit(commit) - new(commit.project, commit.sha) - end - - def initialize(project, sha) - @project = project - @sha = sha + def initialize(commit) + @commit = commit @signature_text, @signed_text = begin - Rugged::Commit.extract_signature(project.repository.rugged, sha) + Rugged::Commit.extract_signature(@commit.project.repository.rugged, @commit.sha) rescue Rugged::OdbError nil end @@ -26,7 +21,7 @@ module Gitlab return @signature if @signature - cached_signature = GpgSignature.find_by(commit_sha: @sha) + cached_signature = GpgSignature.find_by(commit_sha: @commit.sha) return @signature = cached_signature if cached_signature.present? @signature = create_cached_signature! @@ -75,8 +70,8 @@ module Gitlab user_infos = user_infos(gpg_key) { - commit_sha: @sha, - project: @project, + commit_sha: @commit.sha, + project: @commit.project, gpg_key: gpg_key, gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], -- cgit v1.2.1 From 64855c8e30c53004b2e2c2a65f131f8ab7efa41c Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 24 Aug 2017 14:21:30 +0200 Subject: match the committer's email against the gpg key the updated verification of a gpg signature requires the committer's email to also match the user's and the key's emails. --- lib/gitlab/gpg/commit.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index f701897955b..16c8ef563da 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -68,6 +68,7 @@ module Gitlab def attributes(gpg_key) user_infos = user_infos(gpg_key) + verification_status = verification_status(gpg_key) { commit_sha: @commit.sha, @@ -76,12 +77,21 @@ module Gitlab gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], - valid_signature: gpg_signature_valid_signature_value(gpg_key) + valid_signature: verification_status == GpgSignature.verification_statuses[:verified], + verification_status: verification_status } end - def gpg_signature_valid_signature_value(gpg_key) - !!(gpg_key && gpg_key.verified? && verified_signature.valid?) + def verification_status(gpg_key) + if gpg_key && gpg_key.verified_and_belongs_to_email?(@commit.committer_email) && verified_signature.valid? + GpgSignature.verification_statuses[:verified] + elsif gpg_key && gpg_key.verified? && verified_signature.valid? + GpgSignature.verification_statuses[:other_user] + elsif gpg_key + GpgSignature.verification_statuses[:unverified_key] + else + GpgSignature.verification_statuses[:unknown_key] + end end def user_infos(gpg_key) -- cgit v1.2.1 From 2a89037b63967aac0725cf4e122cecbe0c3c5596 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 24 Aug 2017 14:21:38 +0200 Subject: downcase gpg key's emails this is necessary for email comparisons --- lib/gitlab/gpg.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 45e9f9d65ae..025f826e65f 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -39,7 +39,7 @@ module Gitlab fingerprints = CurrentKeyChain.fingerprints_from_key(key) GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| - raw_key.uids.map { |uid| { name: uid.name, email: uid.email } } + raw_key.uids.map { |uid| { name: uid.name, email: uid.email.downcase } } end end end -- cgit v1.2.1 From 00392d929b4553a9ed8e1938cb11f091b79566c9 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 24 Aug 2017 14:21:42 +0200 Subject: add verification_status: same_user_different_email this is used to make a difference between a committer email that belongs to user, where the user used a different email for the gpg key. this means that the user is the same, but a different, unverified email is used for the signature. --- lib/gitlab/gpg/commit.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 16c8ef563da..7aaf3f6aa5b 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -85,6 +85,8 @@ module Gitlab def verification_status(gpg_key) if gpg_key && gpg_key.verified_and_belongs_to_email?(@commit.committer_email) && verified_signature.valid? GpgSignature.verification_statuses[:verified] + elsif gpg_key && gpg_key.verified? && verified_signature.valid? && gpg_key.user.all_emails.include?(@commit.committer_email) + GpgSignature.verification_statuses[:same_user_different_email] elsif gpg_key && gpg_key.verified? && verified_signature.valid? GpgSignature.verification_statuses[:other_user] elsif gpg_key -- cgit v1.2.1 From 7ad7222a5fb2790353e772416090399b9896b129 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 24 Aug 2017 14:21:46 +0200 Subject: simplify if/else with guards --- lib/gitlab/gpg/commit.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 7aaf3f6aa5b..043045ceba7 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -83,16 +83,16 @@ module Gitlab end def verification_status(gpg_key) - if gpg_key && gpg_key.verified_and_belongs_to_email?(@commit.committer_email) && verified_signature.valid? + return GpgSignature.verification_statuses[:unknown_key] unless gpg_key + return GpgSignature.verification_statuses[:unverified_key] unless gpg_key.verified? + return GpgSignature.verification_statuses[:unverified] unless verified_signature.valid? + + if gpg_key.verified_and_belongs_to_email?(@commit.committer_email) GpgSignature.verification_statuses[:verified] - elsif gpg_key && gpg_key.verified? && verified_signature.valid? && gpg_key.user.all_emails.include?(@commit.committer_email) + elsif gpg_key.user.all_emails.include?(@commit.committer_email) GpgSignature.verification_statuses[:same_user_different_email] - elsif gpg_key && gpg_key.verified? && verified_signature.valid? - GpgSignature.verification_statuses[:other_user] - elsif gpg_key - GpgSignature.verification_statuses[:unverified_key] else - GpgSignature.verification_statuses[:unknown_key] + GpgSignature.verification_statuses[:other_user] end end -- cgit v1.2.1 From d6c58a51e5e79071157caa801bda94b2506131d4 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 24 Aug 2017 14:22:19 +0200 Subject: only use symbols instead of enum hash accessor --- lib/gitlab/gpg/commit.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 043045ceba7..a206249ef5a 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -77,22 +77,22 @@ module Gitlab gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], - valid_signature: verification_status == GpgSignature.verification_statuses[:verified], + valid_signature: verification_status == :verified, verification_status: verification_status } end def verification_status(gpg_key) - return GpgSignature.verification_statuses[:unknown_key] unless gpg_key - return GpgSignature.verification_statuses[:unverified_key] unless gpg_key.verified? - return GpgSignature.verification_statuses[:unverified] unless verified_signature.valid? + return :unknown_key unless gpg_key + return :unverified_key unless gpg_key.verified? + return :unverified unless verified_signature.valid? if gpg_key.verified_and_belongs_to_email?(@commit.committer_email) - GpgSignature.verification_statuses[:verified] + :verified elsif gpg_key.user.all_emails.include?(@commit.committer_email) - GpgSignature.verification_statuses[:same_user_different_email] + :same_user_different_email else - GpgSignature.verification_statuses[:other_user] + :other_user end end -- cgit v1.2.1 From 978252a3fa99414446ce887cd28bf2db5b223d44 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 30 Aug 2017 13:27:40 +0200 Subject: use new #verification_status --- lib/gitlab/gpg/commit.rb | 1 - lib/gitlab/gpg/invalid_gpg_signature_updater.rb | 8 +++++++- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index a206249ef5a..86bd9f5b125 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -77,7 +77,6 @@ module Gitlab gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], - valid_signature: verification_status == :verified, verification_status: verification_status } end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index a525ee7a9ee..7bdf6760295 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -6,9 +6,15 @@ module Gitlab end def run + # `OR valid_signature` is for backwards compatibility: legacy records + # that weren't migrated to use the new `#verification_status` have + # `#valid_signature` set instead GpgSignature .select(:id, :commit_sha, :project_id) - .where('gpg_key_id IS NULL OR valid_signature = ?', false) + .where('gpg_key_id IS NULL OR valid_signature = ? OR verification_status <> ?', + false, + GpgSignature.verification_statuses[:verified] + ) .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) .find_each { |sig| sig.gpg_commit.update_signature!(sig) } end -- cgit v1.2.1 From b62c1620ea8d4260bb60303d4825a214c7ed56ee Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 31 Aug 2017 20:43:24 +0200 Subject: drop backwards compatibility for valid_signature --- lib/gitlab/gpg/invalid_gpg_signature_updater.rb | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index 7bdf6760295..e085eab26c9 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -6,15 +6,9 @@ module Gitlab end def run - # `OR valid_signature` is for backwards compatibility: legacy records - # that weren't migrated to use the new `#verification_status` have - # `#valid_signature` set instead GpgSignature .select(:id, :commit_sha, :project_id) - .where('gpg_key_id IS NULL OR valid_signature = ? OR verification_status <> ?', - false, - GpgSignature.verification_statuses[:verified] - ) + .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) .find_each { |sig| sig.gpg_commit.update_signature!(sig) } end -- cgit v1.2.1 From 52e52f4a172ae5aa54eac4a22d98610ec5aea1b0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 5 Sep 2017 12:20:09 +0200 Subject: Make it explicit that workhorse needs artifact path --- lib/gitlab/workhorse.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e5ad9b5a40c..7a94af2f8f1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -121,10 +121,10 @@ module Gitlab ] end - def send_artifacts_entry(build, entry) + def send_artifacts_entry(build, path) params = { 'Archive' => build.artifacts_file.path, - 'Entry' => Base64.encode64(entry.path) + 'Entry' => Base64.encode64(path.to_s) } [ -- cgit v1.2.1 From 3b874414c06156767117b7aa7ae705c7342d887c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 5 Sep 2017 12:27:40 +0200 Subject: Do not use artifacts metadata to access single artifact --- lib/api/jobs.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 41c70a2dcb7..3d71d6bb062 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -98,10 +98,11 @@ module API build = get_build!(params[:job_id]) not_found! unless build.artifacts? - entry = build.artifacts_metadata_entry(params[:artifact_path]) - not_found! unless entry.exists? + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + not_found! unless path.valid? - send_artifacts_entry(build, entry) + send_artifacts_entry(build, path) end desc 'Download the artifacts file from a job' do -- cgit v1.2.1 From d4154ef30f52b30054e8e5d9bbf172d8700e8049 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 5 Sep 2017 13:22:15 +0200 Subject: Do not require API authentication if artifacts are public --- lib/api/jobs.rb | 79 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 36 deletions(-) (limited to 'lib') diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 3d71d6bb062..9e2af071e0a 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -2,12 +2,12 @@ module API class Jobs < Grape::API include PaginationParams - before { authenticate! } - params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + before { authenticate! } + helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', @@ -71,40 +71,6 @@ module API present build, with: Entities::Job end - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.5' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - get ':id/jobs/:job_id/artifacts' do - authorize_read_builds! - - build = get_build!(params[:job_id]) - - present_artifacts!(build.artifacts_file) - end - - desc 'Download a specific file from artifacts archive' do - detail 'This feature was introduced in GitLab 10.0' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - requires :artifact_path, type: String, desc: 'Artifact path' - end - get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do - authorize_read_builds! - - build = get_build!(params[:job_id]) - not_found! unless build.artifacts? - - path = Gitlab::Ci::Build::Artifacts::Path - .new(params[:artifact_path]) - not_found! unless path.valid? - - send_artifacts_entry(build, path) - end - desc 'Download the artifacts file from a job' do detail 'This feature was introduced in GitLab 8.10' end @@ -235,6 +201,47 @@ module API end end + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + before { authenticate_non_get! } + + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + get ':id/jobs/:job_id/artifacts' do + authorize_read_builds! + + build = get_build!(params[:job_id]) + + present_artifacts!(build.artifacts_file) + end + + desc 'Download a specific file from artifacts archive' do + detail 'This feature was introduced in GitLab 10.0' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do + authorize_read_builds! + + build = get_build!(params[:job_id]) + not_found! unless build.artifacts? + + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + not_found! unless path.valid? + + send_artifacts_entry(build, path) + end + end + helpers do def find_build(id) user_project.builds.find_by(id: id.to_i) -- cgit v1.2.1 From b69579742b5379760ccc868d359a9b6d1d225861 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Fri, 25 Aug 2017 12:01:34 +0200 Subject: API: Add GPG key management --- lib/api/entities.rb | 4 +++ lib/api/users.rb | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 0092cc14e74..031dd02c6eb 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -491,6 +491,10 @@ module API expose :user, using: Entities::UserPublic end + class GPGKey < Grape::Entity + expose :id, :key, :created_at + end + class Note < Grape::Entity # Only Issue and MergeRequest have iid NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze diff --git a/lib/api/users.rb b/lib/api/users.rb index 96f47bb618a..82991dc9d16 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -492,6 +492,76 @@ module API destroy_conditionally!(key) end + desc "Get the currently authenticated user's GPG keys" do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + use :pagination + end + get 'gpg_keys' do + present paginate(current_user.gpg_keys), with: Entities::GPGKey + end + + desc 'Get a single GPG key owned by currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + get 'gpg_keys/:key_id' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + present key, with: Entities::GPGKey + end + + desc 'Add a new GPG key to the currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :key, type: String, desc: 'The new GPG key' + end + post 'gpg_keys' do + key = current_user.gpg_keys.new(declared_params) + + if key.save + present key, with: Entities::GPGKey + else + render_validation_error!(key) + end + end + + desc 'Revoke a GPG key owned by currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + post 'gpg_keys/:key_id/revoke' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + key.revoke + status :accepted + end + + desc 'Delete a GPG key from the currently authenticated user' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + delete 'gpg_keys/:key_id' do + key = current_user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + status 204 + key.destroy + end + desc "Get the currently authenticated user's email addresses" do success Entities::Email end -- cgit v1.2.1 From 97371848c5fccb7ba066cc0c50c0d3ed15ec5971 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Fri, 25 Aug 2017 12:24:41 +0200 Subject: API: Add GPG key management for admins --- lib/api/users.rb | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 82991dc9d16..1825c90a23b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -233,6 +233,86 @@ module API destroy_conditionally!(key) end + desc 'Add a GPG key to a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key, type: String, desc: 'The new GPG key' + end + post ':id/gpg_keys' do + authenticated_as_admin! + + user = User.find_by(id: params.delete(:id)) + not_found!('User') unless user + + key = user.gpg_keys.new(declared_params(include_missing: false)) + + if key.save + present key, with: Entities::GPGKey + else + render_validation_error!(key) + end + end + + desc 'Get the GPG keys of a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + success Entities::GPGKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/gpg_keys' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + present paginate(user.gpg_keys), with: Entities::GPGKey + end + + desc 'Delete an existing GPG key from a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + delete ':id/gpg_keys/:key_id' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + key = user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + status 204 + key.destroy + end + + desc 'Revokes an existing GPG key from a specified user. Available only for admins.' do + detail 'This feature was added in GitLab 10.0' + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + post ':id/gpg_keys/:key_id/revoke' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + key = user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + key.revoke + status :accepted + end + desc 'Add an email address to a specified user. Available only for admins.' do success Entities::Email end -- cgit v1.2.1 From f9dd41a775912a7daa8b5929a0100ae5b3c67ca2 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Tue, 5 Sep 2017 14:09:58 +0000 Subject: Do not require the simple_po_parser in the gettext rake task when in a --- lib/tasks/gettext.rake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index f7f2fa2f14c..35ba729c156 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -1,5 +1,4 @@ require "gettext_i18n_rails/tasks" -require 'simple_po_parser' namespace :gettext do # Customize list of translatable files @@ -23,6 +22,8 @@ namespace :gettext do desc 'Lint all po files in `locale/' task lint: :environment do + require 'simple_po_parser' + FastGettext.silence_errors files = Dir.glob(Rails.root.join('locale/*/gitlab.po')) -- cgit v1.2.1 From 0085f5e0aef9e2d6379d2cbc03540c34dd43dc5a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 31 Aug 2017 12:24:25 +0200 Subject: start improving memory use on I/E --- lib/gitlab/import_export/project_tree_restorer.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index cbc8d170936..6fffed68c7e 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -59,8 +59,11 @@ module Gitlab next unless relation_hash_list - relation_hash = create_relation(relation_key, relation_hash_list) - saved << restored_project.append_or_update_attribute(relation_key, relation_hash) + [relation_hash_list].flatten.each_slice(10) do |relation_hash_batch| + + relation_hash = create_relation(relation_key, relation_hash_batch) + saved << restored_project.append_or_update_attribute(relation_key, relation_hash) + end end saved.all? end @@ -77,7 +80,7 @@ module Gitlab @project.update_columns(project_params) @project end - +git che def project_params @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations -- cgit v1.2.1 From 690b9865f48c41f16dd0c70b37dbcc66fee4c67c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 1 Sep 2017 17:20:09 +0200 Subject: some more performance refactoring --- lib/gitlab/import_export/project_tree_restorer.rb | 45 +++++++++++++++++------ 1 file changed, 33 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 6fffed68c7e..a288b7a5b56 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -9,6 +9,7 @@ module Gitlab @user = user @shared = shared @project = project + @project_id = project.id end def restore @@ -48,24 +49,42 @@ module Gitlab # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project. def create_relations - saved = [] + @saved = [] default_relation_list.each do |relation| next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + if relation.is_a?(Hash) + create_sub_relations(relation, @tree_hash) + else + relation_key = relation.is_a?(Hash) ? relation.keys.first : relation + relation_hash_list = @tree_hash[relation_key.to_s] + save_relation_hash(relation_hash_list, relation_key, saved) + end - create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) - relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - relation_hash_list = @tree_hash[relation_key.to_s] + # relation_key = relation.is_a?(Hash) ? relation.keys.first : relation + # relation_hash_list = @tree_hash[relation_key.to_s] + # + # next unless relation_hash_list + # + # if relation_hash_list.is_a?(Array) + # [relation_hash_list].flatten.each_slice(15) do |relation_hash_batch| + # save_relation_hash(relation_hash_batch, relation_key, saved) + # end + # else + # save_relation_hash(relation_hash_list, relation_key, saved) + # end - next unless relation_hash_list + end + @saved.all? + end - [relation_hash_list].flatten.each_slice(10) do |relation_hash_batch| + def save_relation_hash(relation_hash_batch, relation_key) + relation_hash = create_relation(relation_key, relation_hash_batch) + @saved << restored_project.append_or_update_attribute(relation_key, relation_hash) + @restored_project = nil + @project = nil - relation_hash = create_relation(relation_key, relation_hash_batch) - saved << restored_project.append_or_update_attribute(relation_key, relation_hash) - end - end - saved.all? + @project = Project.find_by_id(@project_id) end def default_relation_list @@ -80,7 +99,7 @@ module Gitlab @project.update_columns(project_params) @project end -git che + def project_params @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations @@ -110,6 +129,8 @@ git che relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end + + save_relation_hash(relation_item, relation_key) end end -- cgit v1.2.1 From 025a268881d5441c191474759a84c3a28c6d1965 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Sat, 2 Sep 2017 16:17:41 +0200 Subject: more refactoring --- lib/gitlab/import_export/project_tree_restorer.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index a288b7a5b56..b53de2a33e1 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -80,6 +80,7 @@ module Gitlab def save_relation_hash(relation_hash_batch, relation_key) relation_hash = create_relation(relation_key, relation_hash_batch) + @saved << restored_project.append_or_update_attribute(relation_key, relation_hash) @restored_project = nil @project = nil @@ -115,7 +116,8 @@ module Gitlab # issue, finds any subrelations such as notes, creates them and assign them back to the hash # # Recursively calls this method if the sub-relation is a hash containing more sub-relations - def create_sub_relations(relation, tree_hash) + def create_sub_relations(relation, tree_hash, save = true) + tree_hash = tree_hash.dup relation_key = relation.keys.first.to_s return if tree_hash[relation_key].blank? @@ -124,13 +126,13 @@ module Gitlab # We just use author to get the user ID, do not attempt to create an instance. next if sub_relation == :author - create_sub_relations(sub_relation, relation_item) if sub_relation.is_a?(Hash) + create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash) relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end - save_relation_hash(relation_item, relation_key) + save_relation_hash([relation_item], relation_key) if save end end -- cgit v1.2.1 From a54c50d1469737372398380f819223062bc27f21 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Sun, 3 Sep 2017 12:00:07 +0200 Subject: remove relation after used from the hash tree --- lib/gitlab/import_export/project_tree_restorer.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index b53de2a33e1..3c9598887bf 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -57,7 +57,7 @@ module Gitlab else relation_key = relation.is_a?(Hash) ? relation.keys.first : relation relation_hash_list = @tree_hash[relation_key.to_s] - save_relation_hash(relation_hash_list, relation_key, saved) + save_relation_hash(relation_hash_list, relation_key, @saved) end @@ -82,10 +82,7 @@ module Gitlab relation_hash = create_relation(relation_key, relation_hash_batch) @saved << restored_project.append_or_update_attribute(relation_key, relation_hash) - @restored_project = nil - @project = nil - - @project = Project.find_by_id(@project_id) + @restored_project = Project.find_by_id(@project_id) end def default_relation_list @@ -117,7 +114,6 @@ module Gitlab # # Recursively calls this method if the sub-relation is a hash containing more sub-relations def create_sub_relations(relation, tree_hash, save = true) - tree_hash = tree_hash.dup relation_key = relation.keys.first.to_s return if tree_hash[relation_key].blank? @@ -133,6 +129,7 @@ module Gitlab end save_relation_hash([relation_item], relation_key) if save + tree_hash.delete(relation_key) if save end end -- cgit v1.2.1 From 4a9dcfdc721ff6dd5926be5f20a75a691a5e85f0 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Sun, 3 Sep 2017 19:16:18 +0200 Subject: enable AR caching, refactor tree hash loop --- lib/gitlab/import_export/project_tree_restorer.rb | 39 ++++++++++++++++------- 1 file changed, 27 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 3c9598887bf..eeb3024c83a 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -23,8 +23,10 @@ module Gitlab @project_members = @tree_hash.delete('project_members') - ActiveRecord::Base.no_touching do - create_relations + ActiveRecord::Base.uncached do + ActiveRecord::Base.no_touching do + create_relations + end end rescue => e @shared.error(e) @@ -52,6 +54,7 @@ module Gitlab @saved = [] default_relation_list.each do |relation| next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + if relation.is_a?(Hash) create_sub_relations(relation, @tree_hash) else @@ -82,6 +85,10 @@ module Gitlab relation_hash = create_relation(relation_key, relation_hash_batch) @saved << restored_project.append_or_update_attribute(relation_key, relation_hash) + @restored_project = nil + @project = nil + relation_hash = nil + relation_hash_batch = nil @restored_project = Project.find_by_id(@project_id) end @@ -117,20 +124,28 @@ module Gitlab relation_key = relation.keys.first.to_s return if tree_hash[relation_key].blank? - [tree_hash[relation_key]].flatten.each do |relation_item| - relation.values.flatten.each do |sub_relation| - # We just use author to get the user ID, do not attempt to create an instance. - next if sub_relation == :author + tree_array = [tree_hash[relation_key]].flatten - create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash) + while relation_item = tree_array.shift + relation.values.flatten.each do |sub_relation| + # We just use author to get the user ID, do not attempt to create an instance. + next if sub_relation == :author - relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) - relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? - end + create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash) - save_relation_hash([relation_item], relation_key) if save - tree_hash.delete(relation_key) if save + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? + end + + if save + save_relation_hash([relation_item], relation_key) + end + + tree_hash[relation_key].delete(relation_item) if save + relation_item = nil end + + tree_hash.delete(relation_key) if save end def assign_relation_hash(relation_item, sub_relation) -- cgit v1.2.1 From d3348474d4b611ee8b99d860c3a0f99b0d4f9b60 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Sun, 3 Sep 2017 20:01:14 +0200 Subject: some more refactoring --- lib/gitlab/import_export/project_tree_restorer.rb | 32 +++++++++++------------ 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index eeb3024c83a..0d26f21d766 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -60,7 +60,7 @@ module Gitlab else relation_key = relation.is_a?(Hash) ? relation.keys.first : relation relation_hash_list = @tree_hash[relation_key.to_s] - save_relation_hash(relation_hash_list, relation_key, @saved) + save_relation_hash(relation_hash_list, relation_key) end @@ -85,10 +85,6 @@ module Gitlab relation_hash = create_relation(relation_key, relation_hash_batch) @saved << restored_project.append_or_update_attribute(relation_key, relation_hash) - @restored_project = nil - @project = nil - relation_hash = nil - relation_hash_batch = nil @restored_project = Project.find_by_id(@project_id) end @@ -127,27 +123,29 @@ module Gitlab tree_array = [tree_hash[relation_key]].flatten while relation_item = tree_array.shift - relation.values.flatten.each do |sub_relation| - # We just use author to get the user ID, do not attempt to create an instance. - next if sub_relation == :author - - create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash) - - relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) - relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? - end + process_sub_relation(relation, relation_item) if save save_relation_hash([relation_item], relation_key) + tree_hash[relation_key].delete(relation_item) end - - tree_hash[relation_key].delete(relation_item) if save - relation_item = nil end tree_hash.delete(relation_key) if save end + def process_sub_relation(relation, relation_item) + relation.values.flatten.each do |sub_relation| + # We just use author to get the user ID, do not attempt to create an instance. + next if sub_relation == :author + + create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash) + + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? + end + end + def assign_relation_hash(relation_item, sub_relation) if sub_relation.is_a?(Hash) relation_hash = relation_item[sub_relation.keys.first.to_s] -- cgit v1.2.1 From 8ff9fc55cf876a61fcfde421b71d204d9fd33723 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Sun, 3 Sep 2017 20:51:50 +0200 Subject: some more refactoring --- lib/gitlab/import_export/project_tree_restorer.rb | 21 ++++----------------- lib/gitlab/import_export/shared.rb | 2 +- 2 files changed, 5 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 0d26f21d766..82ca24fd2f8 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -23,8 +23,8 @@ module Gitlab @project_members = @tree_hash.delete('project_members') - ActiveRecord::Base.uncached do - ActiveRecord::Base.no_touching do + ActiveRecord::Base.uncached do + ActiveRecord::Base.no_touching do create_relations end end @@ -62,21 +62,6 @@ module Gitlab relation_hash_list = @tree_hash[relation_key.to_s] save_relation_hash(relation_hash_list, relation_key) end - - - # relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - # relation_hash_list = @tree_hash[relation_key.to_s] - # - # next unless relation_hash_list - # - # if relation_hash_list.is_a?(Array) - # [relation_hash_list].flatten.each_slice(15) do |relation_hash_batch| - # save_relation_hash(relation_hash_batch, relation_key, saved) - # end - # else - # save_relation_hash(relation_hash_list, relation_key, saved) - # end - end @saved.all? end @@ -123,12 +108,14 @@ module Gitlab tree_array = [tree_hash[relation_key]].flatten while relation_item = tree_array.shift + Project.transaction do process_sub_relation(relation, relation_item) if save save_relation_hash([relation_item], relation_key) tree_hash[relation_key].delete(relation_item) end + end end tree_hash.delete(relation_key) if save diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 5d6de8bc475..9fd0b709ef2 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -16,7 +16,7 @@ module Gitlab error_out(error.message, caller[0].dup) @errors << error.message # Debug: - Rails.logger.error(error.backtrace) + Rails.logger.error(error.backtrace.join("\n")) end private -- cgit v1.2.1 From be99f82478e023e41a543a0b9348e27d56fb3c13 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 5 Sep 2017 11:02:34 +0200 Subject: started refactoring reader --- lib/gitlab/import_export/reader.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index eb7f5120592..4f061959047 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -18,7 +18,9 @@ module Gitlab attributes = @attributes_finder.find(:project) project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {} - project_attributes.merge(include: build_hash(@tree)) + build_hash(@tree).each do |sub_hash| + yield(project_attributes.merge(include: sub_hash)) + end rescue => e @shared.error(e) false -- cgit v1.2.1 From 4119206f76d802413850dc9fe1fa4715c3bd6fc0 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 5 Sep 2017 17:10:57 +0200 Subject: fix export performance of CI builds --- lib/gitlab/import_export/import_export.yml | 1 + lib/gitlab/import_export/reader.rb | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 78795dd3d92..ec73846d844 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -116,6 +116,7 @@ excluded_attributes: statuses: - :trace - :token + - :when push_event_payload: - :event_id diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 4f061959047..eb7f5120592 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -18,9 +18,7 @@ module Gitlab attributes = @attributes_finder.find(:project) project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {} - build_hash(@tree).each do |sub_hash| - yield(project_attributes.merge(include: sub_hash)) - end + project_attributes.merge(include: build_hash(@tree)) rescue => e @shared.error(e) false -- cgit v1.2.1 From d8d05e5f284a3e3324ee51149ff278ba1dfbd43c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 5 Sep 2017 17:24:57 +0200 Subject: refactor code a little --- lib/gitlab/import_export/project_tree_restorer.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 82ca24fd2f8..7f24eae7ef8 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -10,6 +10,7 @@ module Gitlab @shared = shared @project = project @project_id = project.id + @saved = [] end def restore @@ -51,7 +52,6 @@ module Gitlab # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project. def create_relations - @saved = [] default_relation_list.each do |relation| next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? @@ -59,10 +59,10 @@ module Gitlab create_sub_relations(relation, @tree_hash) else relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - relation_hash_list = @tree_hash[relation_key.to_s] - save_relation_hash(relation_hash_list, relation_key) + save_relation_hash(@tree_hash[relation_key.to_s], relation_key) end end + @saved.all? end @@ -70,6 +70,8 @@ module Gitlab relation_hash = create_relation(relation_key, relation_hash_batch) @saved << restored_project.append_or_update_attribute(relation_key, relation_hash) + + # Restore the project again, extra query but let us skip holding the AR objects in memory @restored_project = Project.find_by_id(@project_id) end @@ -107,10 +109,17 @@ module Gitlab tree_array = [tree_hash[relation_key]].flatten + # Avoid keeping a possible heavy object in memory once we are done with it while relation_item = tree_array.shift + # The transaction at this level is less speedy than one single transaction + # But we can't have it in the upper level or GC won't get rid of the AR objects + # after we save the batch. Project.transaction do process_sub_relation(relation, relation_item) + # For every subrelation that hangs from Project, save the associated records alltogether + # This effectively batches all records per subrelation item, only keeping those in memory + # We have to keep in mind that more batch granularity << Memory, but >> Slowness if save save_relation_hash([relation_item], relation_key) tree_hash[relation_key].delete(relation_item) -- cgit v1.2.1 From e10437de407a0782cd49cf5f341517c5b17fdd5e Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Tue, 5 Sep 2017 17:29:45 +0000 Subject: Migrate Gitlab::Git::Repository#find_branch to Gitaly --- lib/gitlab/git/repository.rb | 18 +++++++++++------- lib/gitlab/gitaly_client/ref_service.rb | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f7577b02d5d..75d4efc0bc5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -134,15 +134,19 @@ module Gitlab # This is to work around a bug in libgit2 that causes in-memory refs to # be stale/invalid when packed-refs is changed. # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/474 def find_branch(name, force_reload = false) - reload_rugged if force_reload + gitaly_migrate(:find_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.find_branch(name) + else + reload_rugged if force_reload - rugged_ref = rugged.branches[name] - if rugged_ref - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rugged_ref = rugged.branches[name] + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end + end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8c0008c6971..a1a25cf2079 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -78,6 +78,20 @@ module Gitlab raise ArgumentError, e.message end + def find_branch(branch_name) + request = Gitaly::DeleteBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(branch_name) + ) + + response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) + branch = response.branch + return unless branch + + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit) + end + private def consume_refs_response(response) -- cgit v1.2.1 From 104f221b601621cdfdd083f82780188c84190a1c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 5 Sep 2017 21:06:27 +0200 Subject: fix specs --- lib/gitlab/import_export/project_tree_restorer.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 7f24eae7ef8..d365e9ad40b 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -72,7 +72,7 @@ module Gitlab @saved << restored_project.append_or_update_attribute(relation_key, relation_hash) # Restore the project again, extra query but let us skip holding the AR objects in memory - @restored_project = Project.find_by_id(@project_id) + @restored_project = Project.find(@project_id) end def default_relation_list @@ -153,14 +153,12 @@ module Gitlab end def create_relation(relation, relation_hash_list) - relation_type = relation.to_sym - relation_array = [relation_hash_list].flatten.map do |relation_hash| - Gitlab::ImportExport::RelationFactory.create(relation_sym: relation_type, - relation_hash: parsed_relation_hash(relation_hash, relation_type), + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), members_mapper: members_mapper, user: @user, - project: restored_project) + project: @restored_project) end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first -- cgit v1.2.1 From 8323e55b6c0d59627c584b49e70d4f6ccfd3c8f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Thu, 31 Aug 2017 18:00:40 -0300 Subject: Return a value to check if redis is available on /internal/check --- lib/api/helpers/internal_helpers.rb | 9 +++++++++ lib/api/internal.rb | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index f57ff0f2632..4c0db4d42b1 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -46,6 +46,15 @@ module API ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end + def redis_ping + result = Gitlab::Redis::SharedState.with { |redis| redis.ping } + + result == 'PONG' + rescue => e + Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") + false + end + private def set_project diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 622bd9650e4..85b66593175 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -88,7 +88,8 @@ module API { api_version: API.version, gitlab_version: Gitlab::VERSION, - gitlab_rev: Gitlab::REVISION + gitlab_rev: Gitlab::REVISION, + redis: redis_ping } end -- cgit v1.2.1 From ff3eeffa083e6f2299db9c622611762028578bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Thu, 31 Aug 2017 18:01:07 -0300 Subject: Implement /internal/pre-receive for shell operations --- lib/api/internal.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 85b66593175..c0fef56378f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -143,6 +143,14 @@ module API { success: true, recovery_codes: codes } end + post '/pre_receive' do + status 200 + + reference_counter_increased = Gitlab::ReferenceCounter.new(params[:gl_repository]).increase + + { reference_counter_increased: reference_counter_increased } + end + post "/notify_post_receive" do status 200 -- cgit v1.2.1 From 7e38cc9cc215682fc20a7c036dee763df4409198 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 6 Sep 2017 08:56:38 +0200 Subject: small refactor --- lib/gitlab/import_export/project_tree_restorer.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index d365e9ad40b..f8775729bef 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -10,7 +10,7 @@ module Gitlab @shared = shared @project = project @project_id = project.id - @saved = [] + @saved = true end def restore @@ -63,16 +63,16 @@ module Gitlab end end - @saved.all? + @saved end def save_relation_hash(relation_hash_batch, relation_key) relation_hash = create_relation(relation_key, relation_hash_batch) - @saved << restored_project.append_or_update_attribute(relation_key, relation_hash) + @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) - # Restore the project again, extra query but let us skip holding the AR objects in memory - @restored_project = Project.find(@project_id) + # Restore the project again, extra query that skips holding the AR objects in memory + @restored_project.reload end def default_relation_list -- cgit v1.2.1 From 89ca01eb1a2eac5472f3957bb425700dcac82a7a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 6 Sep 2017 10:09:24 +0200 Subject: fix spec failures --- lib/gitlab/import_export/project_tree_restorer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index f8775729bef..4c17f13b851 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -72,7 +72,7 @@ module Gitlab @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) # Restore the project again, extra query that skips holding the AR objects in memory - @restored_project.reload + @restored_project = Project.find(@project_id) end def default_relation_list -- cgit v1.2.1 From e5f91e524628faab81c1d61ebcaa00eaf0954ff6 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 6 Sep 2017 10:16:11 +0200 Subject: refactor code based on feedback --- lib/gitlab/import_export/project_tree_restorer.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4c17f13b851..dec8a4c8cab 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -53,11 +53,9 @@ module Gitlab # Finally, it updates each attribute in the newly imported project. def create_relations default_relation_list.each do |relation| - next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? - if relation.is_a?(Hash) create_sub_relations(relation, @tree_hash) - else + elsif @tree_hash[relation.to_s].present? relation_key = relation.is_a?(Hash) ? relation.keys.first : relation save_relation_hash(@tree_hash[relation_key.to_s], relation_key) end -- cgit v1.2.1 From ef4b3a39bc705dfa47762868ea064d2dbadc16e6 Mon Sep 17 00:00:00 2001 From: Ashley Dumaine Date: Mon, 28 Aug 2017 17:38:06 -0400 Subject: Add functionality to collapse outdated diff comments regardless of discussion resolution --- lib/api/entities.rb | 1 + lib/api/projects.rb | 2 ++ lib/api/v3/entities.rb | 1 + lib/api/v3/projects.rb | 7 ++++--- 4 files changed, 8 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 031dd02c6eb..2799fec9cb6 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -119,6 +119,7 @@ module API expose :archived?, as: :archived expose :visibility expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } + expose :collapse_outdated_diff_comments expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 4845242a173..558eace80f3 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -16,6 +16,7 @@ module API optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :collapse_outdated_diff_comments, type: Boolean, desc: 'Collapse outdated diffs regardless of discussion resolution' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' @@ -236,6 +237,7 @@ module API at_least_one_of_ce = [ :jobs_enabled, + :collapse_outdated_diff_comments, :container_registry_enabled, :default_branch, :description, diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index a9a35f2a4bd..c00df7d00de 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -64,6 +64,7 @@ module API expose :owner, using: ::API::Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :collapse_outdated_diff_comments expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 449876c10d9..2a8906587ce 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -18,6 +18,7 @@ module API optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :collapse_outdated_diff_comments, type: Boolean, desc: 'Collapse outdated diffs regardless of discussion resolution' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' @@ -296,9 +297,9 @@ module API use :optional_params at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, - :shared_runners_enabled, :container_registry_enabled, - :lfs_enabled, :public, :visibility_level, :public_builds, - :request_access_enabled, :only_allow_merge_if_build_succeeds, + :shared_runners_enabled, :collapse_outdated_diff_comments, + :container_registry_enabled, :lfs_enabled, :public, :visibility_level, + :public_builds, :request_access_enabled, :only_allow_merge_if_build_succeeds, :only_allow_merge_if_all_discussions_are_resolved, :path, :default_branch end -- cgit v1.2.1 From a3f76b76a4b8db85c6fa557a5e801dcea7195735 Mon Sep 17 00:00:00 2001 From: Ashley Dumaine Date: Fri, 1 Sep 2017 18:39:22 -0400 Subject: change collapse to resolve and comments to discussions --- lib/api/entities.rb | 2 +- lib/api/projects.rb | 4 ++-- lib/api/v3/entities.rb | 2 +- lib/api/v3/projects.rb | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 2799fec9cb6..9114b69606b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -119,7 +119,7 @@ module API expose :archived?, as: :archived expose :visibility expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } - expose :collapse_outdated_diff_comments + expose :resolve_outdated_diff_discussions expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 558eace80f3..7dc19788462 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -16,7 +16,7 @@ module API optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' - optional :collapse_outdated_diff_comments, type: Boolean, desc: 'Collapse outdated diffs regardless of discussion resolution' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' @@ -237,7 +237,7 @@ module API at_least_one_of_ce = [ :jobs_enabled, - :collapse_outdated_diff_comments, + :resolve_outdated_diff_discussions, :container_registry_enabled, :default_branch, :description, diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c00df7d00de..ac47a713966 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -64,7 +64,7 @@ module API expose :owner, using: ::API::Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :collapse_outdated_diff_comments + expose :resolve_outdated_diff_discussions expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 2a8906587ce..74df246bdfe 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -18,7 +18,7 @@ module API optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' - optional :collapse_outdated_diff_comments, type: Boolean, desc: 'Collapse outdated diffs regardless of discussion resolution' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' @@ -297,7 +297,7 @@ module API use :optional_params at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, - :shared_runners_enabled, :collapse_outdated_diff_comments, + :shared_runners_enabled, :resolve_outdated_diff_discussions, :container_registry_enabled, :lfs_enabled, :public, :visibility_level, :public_builds, :request_access_enabled, :only_allow_merge_if_build_succeeds, :only_allow_merge_if_all_discussions_are_resolved, :path, -- cgit v1.2.1 From 9e1f8ac2a3edb32f672ef2ad5424e99425f67c92 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 6 Sep 2017 11:11:02 +0200 Subject: refactored code --- lib/gitlab/import_export/project_tree_restorer.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index dec8a4c8cab..3bc095a99a9 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -56,8 +56,7 @@ module Gitlab if relation.is_a?(Hash) create_sub_relations(relation, @tree_hash) elsif @tree_hash[relation.to_s].present? - relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - save_relation_hash(@tree_hash[relation_key.to_s], relation_key) + save_relation_hash(@tree_hash[relation.to_s], relation) end end @@ -101,7 +100,7 @@ module Gitlab # issue, finds any subrelations such as notes, creates them and assign them back to the hash # # Recursively calls this method if the sub-relation is a hash containing more sub-relations - def create_sub_relations(relation, tree_hash, save = true) + def create_sub_relations(relation, tree_hash, save: true) relation_key = relation.keys.first.to_s return if tree_hash[relation_key].blank? @@ -133,7 +132,7 @@ module Gitlab # We just use author to get the user ID, do not attempt to create an instance. next if sub_relation == :author - create_sub_relations(sub_relation, relation_item, false) if sub_relation.is_a?(Hash) + create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash) relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? -- cgit v1.2.1 From 80b3dcc777c72ee1853b4228baf6310ccd5e44d8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 6 Sep 2017 11:20:12 +0200 Subject: Extract job artifacts API code to a separate file --- lib/api/api.rb | 1 + lib/api/helpers.rb | 12 ++++++ lib/api/job_artifacts.rb | 80 ++++++++++++++++++++++++++++++++++++ lib/api/jobs.rb | 105 +++-------------------------------------------- 4 files changed, 99 insertions(+), 99 deletions(-) create mode 100644 lib/api/job_artifacts.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 94df543853b..1405a5d0f0e 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -108,6 +108,7 @@ module API mount ::API::Internal mount ::API::Issues mount ::API::Jobs + mount ::API::JobArtifacts mount ::API::Keys mount ::API::Labels mount ::API::Lint diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index f9ce1165544..6fa1527461f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -128,6 +128,10 @@ module API merge_request end + def find_build!(id) + user_project.builds.find(id.to_i) + end + def authenticate! unauthorized! unless current_user && can?(initial_current_user, :access_api) end @@ -160,6 +164,14 @@ module API authorize! :admin_project, user_project end + def authorize_read_builds! + authorize! :read_build, user_project + end + + def authorize_update_builds! + authorize! :update_build, user_project + end + def require_gitlab_workhorse! unless env['HTTP_GITLAB_WORKHORSE'].present? forbidden!('Request should be executed via GitLab Workhorse') diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb new file mode 100644 index 00000000000..e0c5449e65e --- /dev/null +++ b/lib/api/job_artifacts.rb @@ -0,0 +1,80 @@ +module API + class JobArtifacts < Grape::API + before { authenticate_non_get! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the job' + end + get ':id/jobs/artifacts/:ref_name/download', + requirements: { ref_name: /.+/ } do + authorize_read_builds! + + builds = user_project.latest_successful_builds_for(params[:ref_name]) + latest_build = builds.find_by!(name: params[:job]) + + present_artifacts!(latest_build.artifacts_file) + end + + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + get ':id/jobs/:job_id/artifacts' do + authorize_read_builds! + + build = find_build!(params[:job_id]) + + present_artifacts!(build.artifacts_file) + end + + desc 'Download a specific file from artifacts archive' do + detail 'This feature was introduced in GitLab 10.0' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do + authorize_read_builds! + + build = find_build!(params[:job_id]) + not_found! unless build.artifacts? + + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + not_found! unless path.valid? + + send_artifacts_entry(build, path) + end + + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + post ':id/jobs/:job_id/artifacts/keep' do + authorize_update_builds! + + build = find_build!(params[:job_id]) + authorize!(:update_build, build) + return not_found!(build) unless build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Job + end + end + end +end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 9e2af071e0a..19161aabd37 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -66,28 +66,11 @@ module API get ':id/jobs/:job_id' do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) present build, with: Entities::Job end - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.10' - end - params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the job' - end - get ':id/jobs/artifacts/:ref_name/download', - requirements: { ref_name: /.+/ } do - authorize_read_builds! - - builds = user_project.latest_successful_builds_for(params[:ref_name]) - latest_build = builds.find_by!(name: params[:job]) - - present_artifacts!(latest_build.artifacts_file) - end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. @@ -98,7 +81,7 @@ module API get ':id/jobs/:job_id/trace' do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' @@ -117,7 +100,7 @@ module API post ':id/jobs/:job_id/cancel' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) build.cancel @@ -134,7 +117,7 @@ module API post ':id/jobs/:job_id/retry' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) return forbidden!('Job is not retryable') unless build.retryable? @@ -152,7 +135,7 @@ module API post ':id/jobs/:job_id/erase' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) return forbidden!('Job is not erasable!') unless build.erasable? @@ -160,25 +143,6 @@ module API present build, with: Entities::Job end - desc 'Keep the artifacts to prevent them from being deleted' do - success Entities::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - post ':id/jobs/:job_id/artifacts/keep' do - authorize_update_builds! - - build = get_build!(params[:job_id]) - authorize!(:update_build, build) - return not_found!(build) unless build.artifacts? - - build.keep_artifacts! - - status 200 - present build, with: Entities::Job - end - desc 'Trigger a manual job' do success Entities::Job detail 'This feature was added in GitLab 8.11' @@ -189,7 +153,7 @@ module API post ":id/jobs/:job_id/play" do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? @@ -201,56 +165,7 @@ module API end end - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - before { authenticate_non_get! } - - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.5' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - get ':id/jobs/:job_id/artifacts' do - authorize_read_builds! - - build = get_build!(params[:job_id]) - - present_artifacts!(build.artifacts_file) - end - - desc 'Download a specific file from artifacts archive' do - detail 'This feature was introduced in GitLab 10.0' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - requires :artifact_path, type: String, desc: 'Artifact path' - end - get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do - authorize_read_builds! - - build = get_build!(params[:job_id]) - not_found! unless build.artifacts? - - path = Gitlab::Ci::Build::Artifacts::Path - .new(params[:artifact_path]) - not_found! unless path.valid? - - send_artifacts_entry(build, path) - end - end - helpers do - def find_build(id) - user_project.builds.find_by(id: id.to_i) - end - - def get_build!(id) - find_build(id) || not_found! - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? @@ -261,14 +176,6 @@ module API builds.where(status: available_statuses && scope) end - - def authorize_read_builds! - authorize! :read_build, user_project - end - - def authorize_update_builds! - authorize! :update_build, user_project - end end end end -- cgit v1.2.1 From c922fb4b68e3d57bf5328b6825040f1871a48b66 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 6 Sep 2017 11:31:08 +0200 Subject: Respond with a bad request if artifact path is invalid --- lib/api/helpers.rb | 2 +- lib/api/job_artifacts.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 6fa1527461f..e646c63467a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -222,7 +222,7 @@ module API def bad_request!(attribute) message = ["400 (Bad request)"] - message << "\"" + attribute.to_s + "\" not given" + message << "\"" + attribute.to_s + "\" not given" if attribute render_api_error!(message.join(' '), 400) end diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index e0c5449e65e..2a8fa7659bf 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -52,7 +52,7 @@ module API path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) - not_found! unless path.valid? + bad_request! unless path.valid? send_artifacts_entry(build, path) end -- cgit v1.2.1 From b03d9e506d88cbfa13d34e12e0595dbb92f46e7d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 6 Sep 2017 11:42:14 +0200 Subject: Remove unneeded string interpolation from entry class --- lib/gitlab/ci/build/artifacts/metadata/entry.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 321ddfd8c29..22941d48edf 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -113,7 +113,7 @@ module Gitlab end def inspect - "#{self.class.name}: #{self.to_s}" + "#{self.class.name}: #{self}" end private -- cgit v1.2.1 From ec7a12da818898b278a3e47a9b96ccebafbe5b4c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 6 Sep 2017 14:45:28 +0200 Subject: Revert moving authorization hook in jobs API --- lib/api/jobs.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 19161aabd37..3c1c412ba42 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -2,12 +2,12 @@ module API class Jobs < Grape::API include PaginationParams + before { authenticate! } + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - before { authenticate! } - helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', -- cgit v1.2.1 From 966b1128d884a318dad4277e23368334fe67e836 Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Sat, 29 Jul 2017 11:04:42 -0400 Subject: WIP: refactor the first-contributor to Issuable this will remove the need make N queries (per-note) at the cost of having to mark notes with an attribute this opens up the possibility for other special roles for notes --- lib/gitlab/access.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 4714ab18cc1..b4012ebbb99 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -67,10 +67,14 @@ module Gitlab def protection_values protection_options.values end + + def human_access(access) + options_with_owner.key(access) + end end def human_access - Gitlab::Access.options_with_owner.key(access_field) + Gitlab::Access.human_access(access_field) end def owner? -- cgit v1.2.1 From bca72f5906ed38dc231ef066231238758c1cb42d Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Sun, 3 Sep 2017 07:45:44 -0400 Subject: wip: fake its a binary diff --- lib/api/commits.rb | 2 +- lib/api/entities.rb | 3 ++- lib/gitlab/encoding_helper.rb | 16 ++++++++++++---- lib/gitlab/git/diff.rb | 16 +++++++++++++++- 4 files changed, 30 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/api/commits.rb b/lib/api/commits.rb index ea78737288a..4b8d248f5f7 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -104,7 +104,7 @@ module API not_found! 'Commit' unless commit - commit.raw_diffs.to_a + present commit.raw_diffs.to_a, with: Entities::RepoDiff end desc "Get a commit's comments" do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9114b69606b..0c63dc345c2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -291,10 +291,11 @@ module API end class RepoDiff < Grape::Entity - expose :old_path, :new_path, :a_mode, :b_mode, :diff + expose :old_path, :new_path, :a_mode, :b_mode expose :new_file?, as: :new_file expose :renamed_file?, as: :renamed_file expose :deleted_file?, as: :deleted_file + expose :diff end class ProtectedRefAccess < Grape::Entity diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 8ddc91e341d..c5e173ba55a 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -13,6 +13,8 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 ENCODING_CONFIDENCE_THRESHOLD = 50 + # + # def encode!(message) return nil unless message.respond_to? :force_encoding @@ -22,20 +24,26 @@ module Gitlab # return message if message type is binary detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary + return message.force_encoding("BINARY") if binary?(message, detect) - # force detected encoding if we have sufficient confidence. if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + # force detected encoding if we have sufficient confidence. message.force_encoding(detect[:encoding]) end # encode and clean the bad chars message.replace clean(message) - rescue + rescue => e + byebug encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" end + def binary?(message, detect=nil) + detect ||= CharlockHolmes::EncodingDetector.detect(message) + detect && detect[:type] == :binary && detect[:confidence] == 100 + end + def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] @@ -50,7 +58,7 @@ module Gitlab clean(message) end end - + private def clean(message) diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index ce3d65062e8..e544f255a5d 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -116,6 +116,13 @@ module Gitlab filtered_opts end + + # Return a binary diff message like: + # + # "Binary files a/file/path and b/file/path differ\n" + def binary_message(old_path, new_path) + "Binary files #{old_path} and #{new_path} differ\n" + end end def initialize(raw_diff, expanded: true) @@ -214,7 +221,14 @@ module Gitlab # binary we're not going to display anything so we skip the size check. return if !patch.delta.binary? && prune_large_patch(patch) - @diff = encode!(strip_diff_headers(patch.to_s)) + diff = strip_diff_headers(patch.to_s) + @diff = if binary?(diff) + # the diff is binary, let's make a message for it + Diff::binary_message(patch.delta.old_file[:path], + patch.delta.new_file[:path]) + else + encode!(diff) + end end def init_from_hash(hash) -- cgit v1.2.1 From c9aa7932152bd10f6fc7b87b3263922aa4b911b7 Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Mon, 4 Sep 2017 13:34:15 -0400 Subject: revert to using a simple representation --- lib/api/entities.rb | 2 +- lib/gitlab/encoding_helper.rb | 17 ++++++++++++----- lib/gitlab/git/blob.rb | 12 ++++-------- lib/gitlab/git/diff.rb | 16 ++++++++-------- 4 files changed, 25 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 0c63dc345c2..1d224d7bc21 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -295,7 +295,7 @@ module API expose :new_file?, as: :new_file expose :renamed_file?, as: :renamed_file expose :deleted_file?, as: :deleted_file - expose :diff + expose :json_safe_diff, as: :diff end class ProtectedRefAccess < Grape::Entity diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index c5e173ba55a..21a14141c87 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -24,7 +24,7 @@ module Gitlab # return message if message type is binary detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if binary?(message, detect) + return message.force_encoding("BINARY") if all_binary?(message, detect) if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD # force detected encoding if we have sufficient confidence. @@ -34,14 +34,21 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) rescue => e - byebug encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" end - def binary?(message, detect=nil) - detect ||= CharlockHolmes::EncodingDetector.detect(message) - detect && detect[:type] == :binary && detect[:confidence] == 100 + def all_binary?(data, detect=nil) + detect ||= CharlockHolmes::EncodingDetector.detect(data) + detect && detect[:type] == :binary + end + + def libgit2_binary?(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + all_binary?(data, detect) end def encode_utf8(message) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 7780f4e4d4f..2e6edb8be0d 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -42,14 +42,6 @@ module Gitlab end end - def binary?(data) - # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks - # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), - # which is what we use below to keep a consistent behavior. - detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) - detect && detect[:type] == :binary - end - # Returns an array of Blob instances, specified in blob_references as # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the # full blob contents are returned. If blob_size_limit >= 0 then each blob will @@ -169,6 +161,10 @@ module Gitlab end end end + + def binary?(data) + EncodingHelper.libgit2_binary?(data) + end end def initialize(options) diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index e544f255a5d..17defe55af0 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -197,6 +197,13 @@ module Gitlab @collapsed = true end + def json_safe_diff + return @diff unless all_binary?(@diff) + + # the diff is binary, let's make a message for it + Diff::binary_message(@old_path, @new_path) + end + private def init_from_rugged(rugged) @@ -221,14 +228,7 @@ module Gitlab # binary we're not going to display anything so we skip the size check. return if !patch.delta.binary? && prune_large_patch(patch) - diff = strip_diff_headers(patch.to_s) - @diff = if binary?(diff) - # the diff is binary, let's make a message for it - Diff::binary_message(patch.delta.old_file[:path], - patch.delta.new_file[:path]) - else - encode!(diff) - end + @diff = encode!(strip_diff_headers(patch.to_s)) end def init_from_hash(hash) -- cgit v1.2.1 From dbaed90c8d3dffb2dd970f1621f551dd322db5ee Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Mon, 4 Sep 2017 15:32:57 -0400 Subject: fix refactoring error with Blob.binary? remove some lint --- lib/gitlab/encoding_helper.rb | 8 ++++---- lib/gitlab/git/blob.rb | 8 ++++---- lib/gitlab/git/diff.rb | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 21a14141c87..8ac756a0b6a 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -14,7 +14,7 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 # - # + # def encode!(message) return nil unless message.respond_to? :force_encoding @@ -33,12 +33,12 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) - rescue => e + rescue encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" end - def all_binary?(data, detect=nil) + def all_binary?(data, detect = nil) detect ||= CharlockHolmes::EncodingDetector.detect(data) detect && detect[:type] == :binary end @@ -65,7 +65,7 @@ module Gitlab clean(message) end end - + private def clean(message) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 2e6edb8be0d..e5391b8bf8a 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -57,6 +57,10 @@ module Gitlab end end + def binary?(data) + EncodingHelper.libgit2_binary?(data) + end + private # Recursive search of blob id by path @@ -161,10 +165,6 @@ module Gitlab end end end - - def binary?(data) - EncodingHelper.libgit2_binary?(data) - end end def initialize(options) diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 17defe55af0..b3237d5496a 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -118,7 +118,7 @@ module Gitlab end # Return a binary diff message like: - # + # # "Binary files a/file/path and b/file/path differ\n" def binary_message(old_path, new_path) "Binary files #{old_path} and #{new_path} differ\n" @@ -201,7 +201,7 @@ module Gitlab return @diff unless all_binary?(@diff) # the diff is binary, let's make a message for it - Diff::binary_message(@old_path, @new_path) + Diff.binary_message(@old_path, @new_path) end private -- cgit v1.2.1 From a6af5522d7cefaa7d08d28448b5059be07328a2f Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Tue, 5 Sep 2017 13:16:08 -0400 Subject: renames ambiguous methods and add spec --- lib/gitlab/encoding_helper.rb | 10 +++++----- lib/gitlab/git/blob.rb | 2 +- lib/gitlab/git/diff.rb | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 8ac756a0b6a..c50417f4d4f 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -24,7 +24,7 @@ module Gitlab # return message if message type is binary detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if all_binary?(message, detect) + return message.force_encoding("BINARY") if detect_binary?(message, detect) if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD # force detected encoding if we have sufficient confidence. @@ -38,17 +38,17 @@ module Gitlab "--broken encoding: #{encoding}" end - def all_binary?(data, detect = nil) + def detect_binary?(data, detect = nil) detect ||= CharlockHolmes::EncodingDetector.detect(data) - detect && detect[:type] == :binary + detect && detect[:type] == :binary && detect[:confidence] == 100 end - def libgit2_binary?(data) + def detect_libgit2_binary?(data) # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), # which is what we use below to keep a consistent behavior. detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) - all_binary?(data, detect) + detect && detect[:type] == :binary end def encode_utf8(message) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index e5391b8bf8a..8d96826f6ee 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -58,7 +58,7 @@ module Gitlab end def binary?(data) - EncodingHelper.libgit2_binary?(data) + EncodingHelper.detect_libgit2_binary?(data) end private diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index b3237d5496a..a23c8cf0dd1 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -120,6 +120,8 @@ module Gitlab # Return a binary diff message like: # # "Binary files a/file/path and b/file/path differ\n" + # This is used when we detect that a diff is binary + # using CharlockHolmes when Rugged treats it as text. def binary_message(old_path, new_path) "Binary files #{old_path} and #{new_path} differ\n" end @@ -198,7 +200,7 @@ module Gitlab end def json_safe_diff - return @diff unless all_binary?(@diff) + return @diff unless detect_binary?(@diff) # the diff is binary, let's make a message for it Diff.binary_message(@old_path, @new_path) -- cgit v1.2.1 From c1f66cc7f0658621f2bfcb09f0269da675f57e84 Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Wed, 6 Sep 2017 09:15:27 -0400 Subject: remove useless comment --- lib/gitlab/encoding_helper.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index c50417f4d4f..7b3483a7f96 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -13,8 +13,6 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 ENCODING_CONFIDENCE_THRESHOLD = 50 - # - # def encode!(message) return nil unless message.respond_to? :force_encoding -- cgit v1.2.1 From dd7434e398d26e5f4c92f73dce4bcf9b7e47e193 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Wed, 23 Aug 2017 12:10:36 +0100 Subject: Adds CommitDescriptionRenderer --- lib/banzai/commit_description_renderer.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 lib/banzai/commit_description_renderer.rb (limited to 'lib') diff --git a/lib/banzai/commit_description_renderer.rb b/lib/banzai/commit_description_renderer.rb new file mode 100644 index 00000000000..9600ab918a4 --- /dev/null +++ b/lib/banzai/commit_description_renderer.rb @@ -0,0 +1,7 @@ +module Banzai + module CommitDescriptionRenderer + def self.render(commit_descriptions, project, user = nil) + ObjectRenderer.new(project, user).render(commit_descriptions, :commit_description) + end + end +end -- cgit v1.2.1 From 4df54f260751a832ebf0b8c18524020d6604994b Mon Sep 17 00:00:00 2001 From: Alexander Keramidas Date: Tue, 29 Aug 2017 11:57:41 +0300 Subject: Profile updates from providers --- lib/gitlab/ldap/user.rb | 4 ++-- lib/gitlab/o_auth/auth_hash.rb | 17 +++++++++++++++-- lib/gitlab/o_auth/user.rb | 32 +++++++++++++++++++++----------- lib/gitlab/saml/user.rb | 2 +- 4 files changed, 39 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 39180dc17d9..3bf27b37ae6 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -36,7 +36,7 @@ module Gitlab end def find_by_email - ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email? + ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email) end def update_user_attributes @@ -60,7 +60,7 @@ module Gitlab ldap_config.block_auto_created_users end - def sync_email_from_provider? + def sync_profile_from_provider? true end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 7d6911a1ab3..1f331b1e91d 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -32,8 +32,21 @@ module Gitlab @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) end - def has_email? - get_info(:email).present? + def location + location = get_info(:address) + if location.is_a?(Hash) + [location.locality.presence, location.country.presence].compact.join(', ') + else + location + end + end + + def has_attribute?(attribute) + if attribute == :location + get_info(:address).present? + else + get_info(attribute).present? + end end private diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e8330917e91..7704bf715e4 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -12,7 +12,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash - update_email + update_profile if sync_profile_from_provider? end def persisted? @@ -184,20 +184,30 @@ module Gitlab } end - def sync_email_from_provider? - auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s + def sync_profile_from_provider? + providers = Gitlab.config.omniauth.sync_profile_from_provider + + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end - def update_email - if auth_hash.has_email? && sync_email_from_provider? - if persisted? - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email - end + def update_profile + user_synced_attributes_metadata = gl_user.user_synced_attributes_metadata || gl_user.build_user_synced_attributes_metadata - gl_user.external_email = true - gl_user.email_provider = auth_hash.provider + UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| + if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) + gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend + user_synced_attributes_metadata.set_attribute_synced(key, true) + else + user_synced_attributes_metadata.set_attribute_synced(key, false) + end end + + user_synced_attributes_metadata.provider = auth_hash.provider + gl_user.user_synced_attributes_metadata = user_synced_attributes_metadata end def log diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 8a7cc690046..0f323a9e8b2 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -40,7 +40,7 @@ module Gitlab end def find_by_email - if auth_hash.has_email? + if auth_hash.has_attribute?(:email) user = ::User.find_by(email: auth_hash.email.downcase) user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user user -- cgit v1.2.1 From 759f34bd0a250cb2cdf1b718837b56bb28fa1939 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Wed, 6 Sep 2017 12:29:14 +0100 Subject: Backport EE fixes to Gitlab::UrlSanitizer to CE --- lib/gitlab/url_sanitizer.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index c81dc7e30d0..9c26490f40f 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -9,7 +9,7 @@ module Gitlab end def self.valid?(url) - return false unless url + return false unless url.present? Addressable::URI.parse(url.strip) @@ -29,13 +29,13 @@ module Gitlab def masked_url url = @url.dup - url.password = "*****" unless url.password.nil? - url.user = "*****" unless url.user.nil? + url.password = "*****" if url.password.present? + url.user = "*****" if url.user.present? url.to_s end def credentials - @credentials ||= { user: @url.user, password: @url.password } + @credentials ||= { user: @url.user.presence, password: @url.password.presence } end def full_url @@ -47,8 +47,8 @@ module Gitlab def generate_full_url return @url unless valid_credentials? @full_url = @url.dup - @full_url.user = credentials[:user] - @full_url.password = credentials[:password] + @full_url.user = credentials[:user].presence + @full_url.password = credentials[:password].presence @full_url end -- cgit v1.2.1 From c8bdb20228b34130c7f0525ad92140702dce1e20 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Wed, 6 Sep 2017 15:20:25 +0100 Subject: Remove blank passwords from sanitized URLs --- lib/gitlab/url_sanitizer.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 9c26490f40f..703adae12cb 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -19,7 +19,12 @@ module Gitlab end def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url.strip) + @url = Addressable::URI.parse(url.to_s.strip) + + %i[user password].each do |symbol| + credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol) + end + @credentials = credentials end @@ -47,8 +52,10 @@ module Gitlab def generate_full_url return @url unless valid_credentials? @full_url = @url.dup - @full_url.user = credentials[:user].presence - @full_url.password = credentials[:password].presence + + @full_url.password = credentials[:password] + @full_url.user = credentials[:user] + @full_url end -- cgit v1.2.1 From 6d8e102c740b75ac9e1d168a84f532f6d9ebaa65 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Wed, 23 Aug 2017 17:53:29 +0100 Subject: Adds cacheless render to Banzai object render --- lib/banzai/commit_description_renderer.rb | 7 ------- lib/banzai/commit_renderer.rb | 11 +++++++++++ lib/banzai/object_renderer.rb | 2 +- lib/banzai/renderer.rb | 4 ++++ 4 files changed, 16 insertions(+), 8 deletions(-) delete mode 100644 lib/banzai/commit_description_renderer.rb create mode 100644 lib/banzai/commit_renderer.rb (limited to 'lib') diff --git a/lib/banzai/commit_description_renderer.rb b/lib/banzai/commit_description_renderer.rb deleted file mode 100644 index 9600ab918a4..00000000000 --- a/lib/banzai/commit_description_renderer.rb +++ /dev/null @@ -1,7 +0,0 @@ -module Banzai - module CommitDescriptionRenderer - def self.render(commit_descriptions, project, user = nil) - ObjectRenderer.new(project, user).render(commit_descriptions, :commit_description) - end - end -end diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb new file mode 100644 index 00000000000..f5ff95e3eb3 --- /dev/null +++ b/lib/banzai/commit_renderer.rb @@ -0,0 +1,11 @@ +module Banzai + module CommitRenderer + ATTRIBUTES = [:description, :title].freeze + + def self.render(commits, project, user = nil) + obj_renderer = ObjectRenderer.new(project, user) + + ATTRIBUTES.each { |attr| obj_renderer.render(commits, attr) } + end + end +end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 2196a92474c..e40556e869c 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -38,7 +38,7 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) # rubocop:disable GitlabSecurity/PublicSend - object.user_visible_reference_count = redacted_data[:visible_reference_count] + object.user_visible_reference_count = redacted_data[:visible_reference_count] if object.respond_to?(:user_visible_reference_count) end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 95d82d17658..ceca9296851 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -36,6 +36,10 @@ module Banzai # The context to use is managed by the object and cannot be changed. # Use #render, passing it the field text, if a custom rendering is needed. def self.render_field(object, field) + unless object.respond_to?(:cached_markdown_fields) + return cacheless_render_field(object, field) + end + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) object.cached_html_for(field) -- cgit v1.2.1 From 235b105c917c16ab14a79ba13280aff4fd9f1cf9 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Aug 2017 15:38:16 +0200 Subject: Finish migration to the new events setup This finishes the procedure for migrating events from the old format into the new format. Code no longer uses the old setup and the database tables used during the migration process are swapped, with the old table being dropped. While the database migration can be reversed this will 1) take a lot of time as data has to be coped around 2) won't restore data in the "events.data" column as we have no way of restoring this. Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/37241 --- lib/api/entities.rb | 2 +- lib/api/v3/entities.rb | 2 +- lib/gitlab/import_export.rb | 2 +- lib/gitlab/import_export/relation_factory.rb | 1 - lib/tasks/gitlab/import_export.rake | 11 +++++++++++ 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 031dd02c6eb..403c91cacc7 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -545,7 +545,7 @@ module API end class Event < Grape::Entity - expose :title, :project_id, :action_name + expose :project_id, :action_name expose :target_id, :target_iid, :target_type, :author_id expose :target_title expose :created_at diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index a9a35f2a4bd..576652ab6d9 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -31,7 +31,7 @@ module API end class Event < Grape::Entity - expose :title, :project_id, :action_name + expose :project_id, :action_name expose :target_id, :target_type, :author_id expose :target_title expose :created_at diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 3470a09eaf0..50ee879129c 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.8'.freeze + VERSION = '0.2.0'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 20580459046..d563a87dcfd 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -69,7 +69,6 @@ module Gitlab reset_tokens! remove_encrypted_attributes! - @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diff_commits if @relation_name == :merge_request_diff set_diff if @relation_name == :merge_request_diff_files end diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index dd1825c8a9e..44074397c05 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -9,5 +9,16 @@ namespace :gitlab do task data: :environment do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end + + desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' + task bump_test_version: :environment do + Dir.mktmpdir do |tmp_dir| + system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + end + + puts "Updated to #{Gitlab::ImportExport.version}" + end end end -- cgit v1.2.1 From 41ef94e777d9c9d10a8b64b1498f57a8e5847e23 Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Mon, 28 Aug 2017 09:31:41 +0200 Subject: Migrate creating/deleting a branch to Gitaly --- lib/github/representation/branch.rb | 2 +- lib/gitlab/git/repository.rb | 37 +++++++++++++++++++++--------- lib/gitlab/gitaly_client/ref_service.rb | 40 ++++++++++++++++++++++++++++++++- lib/gitlab/github_import/importer.rb | 2 +- 4 files changed, 67 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb index c6fa928d565..823e8e9a9c4 100644 --- a/lib/github/representation/branch.rb +++ b/lib/github/representation/branch.rb @@ -41,7 +41,7 @@ module Github def remove!(name) repository.delete_branch(name) - rescue Rugged::ReferenceError => e + rescue Gitlab::Git::Repository::DeleteBranchError => e Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}") end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 75d4efc0bc5..efa13590a2c 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -18,6 +18,7 @@ module Gitlab InvalidBlobName = Class.new(StandardError) InvalidRef = Class.new(StandardError) GitError = Class.new(StandardError) + DeleteBranchError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the storage path, not the storage name @@ -653,10 +654,16 @@ module Gitlab end # Delete the specified branch from the repository - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def delete_branch(branch_name) - rugged.branches.delete(branch_name) + gitaly_migrate(:delete_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.delete_branch(branch_name) + else + rugged.branches.delete(branch_name) + end + end + rescue Rugged::ReferenceError, CommandError => e + raise DeleteBranchError, e end def delete_refs(*ref_names) @@ -681,15 +688,14 @@ module Gitlab # Examples: # create_branch("feature") # create_branch("other-feature", "master") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def create_branch(ref, start_point = "HEAD") - rugged_ref = rugged.branches.create(ref, start_point) - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError => e - raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ - raise InvalidRef.new("Invalid reference #{start_point}") + gitaly_migrate(:create_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.create_branch(ref, start_point) + else + rugged_create_branch(ref, start_point) + end + end end # Delete the specified remote from this repository. @@ -1226,6 +1232,15 @@ module Gitlab false end + def rugged_create_branch(ref, start_point) + rugged_ref = rugged.branches.create(ref, start_point) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError => e + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") + end + def gitaly_copy_gitattributes(revision) gitaly_repository_client.apply_gitattributes(revision) end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index a1a25cf2079..8ef873d5848 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -79,7 +79,7 @@ module Gitlab end def find_branch(branch_name) - request = Gitaly::DeleteBranchRequest.new( + request = Gitaly::FindBranchRequest.new( repository: @gitaly_repo, name: GitalyClient.encode(branch_name) ) @@ -92,6 +92,40 @@ module Gitlab Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit) end + def create_branch(ref, start_point) + request = Gitaly::CreateBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(ref), + start_point: GitalyClient.encode(start_point) + ) + + response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) + + case response.status + when :OK + branch = response.branch + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit) + when :ERR_INVALID + invalid_ref!("Invalid ref name") + when :ERR_EXISTS + invalid_ref!("Branch #{ref} already exists") + when :ERR_INVALID_START_POINT + invalid_ref!("Invalid reference #{start_point}") + else + raise "Unknown response status: #{response.status}" + end + end + + def delete_branch(branch_name) + request = Gitaly::DeleteBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(branch_name) + ) + + GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) + end + private def consume_refs_response(response) @@ -163,6 +197,10 @@ module Gitlab Gitlab::Git::Commit.decorate(@repository, hash) end + + def invalid_ref!(message) + raise Gitlab::Git::Repository::InvalidRef.new(message) + end end end end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 373062b354b..b8c07460ebb 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -166,7 +166,7 @@ module Gitlab def remove_branch(name) project.repository.delete_branch(name) - rescue Rugged::ReferenceError + rescue Gitlab::Git::Repository::DeleteBranchFailed errors << { type: :remove_branch, name: name } end -- cgit v1.2.1 From 2a055c23c27f85db4bc56f07dccca642fc264d57 Mon Sep 17 00:00:00 2001 From: Akihiro Nakashima Date: Fri, 18 Aug 2017 16:09:10 +0900 Subject: Fix indentation level in Wiki's TOC items to regard header level --- lib/banzai/filter/table_of_contents_filter.rb | 47 ++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 8e7084f2543..8cb860c5a92 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -15,6 +15,7 @@ module Banzai # `li` child elements. class TableOfContentsFilter < HTML::Pipeline::Filter PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u + HeaderNode = Struct.new(:level, :href, :text, :children, :parent) def call return doc if context[:no_header_anchors] @@ -23,6 +24,10 @@ module Banzai headers = Hash.new(0) + # root node of header-tree + header_root = HeaderNode.new(0, nil, nil, [], nil) + current_header = header_root + doc.css('h1, h2, h3, h4, h5, h6').each do |node| text = node.text @@ -38,12 +43,38 @@ module Banzai # namespace detection will be automatically handled via javascript (see issue #22781) namespace = "user-content-" href = "#{id}#{uniq}" - push_toc(href, text) + + level = node.name[1].to_i # get this header level + if level == current_header.level + # same as previous + parent = current_header.parent + elsif level > current_header.level + # larger (weaker) than previous + parent = current_header + else + # smaller (stronger) than previous + # search parent + parent = current_header + parent = parent.parent while parent.level >= level + end + + # create header-node and push as child + header_node = HeaderNode.new(level, href, text, [], parent) + parent.children.push(header_node) + current_header = header_node + header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href)) end end - result[:toc] = %Q{
    \n#{result[:toc]}
} unless result[:toc].empty? + # extract header-tree + if header_root.children.length > 0 + result[:toc] = %Q{
    \n} + header_root.children.each do |child| + push_toc(child) + end + result[:toc] << '
' + end doc end @@ -54,8 +85,16 @@ module Banzai %Q{} end - def push_toc(href, text) - result[:toc] << %Q{
  • #{text}
    • \n} + def push_toc(header_node) + result[:toc] << %Q{
  • #{header_node.text}} + if header_node.children.length > 0 + result[:toc] << '
      ' + header_node.children.each do |child| + push_toc(child) + end + result[:toc] << '
    ' + end + result[:toc] << '
    • \n' end end end -- cgit v1.2.1 From b43aefbd9d606d01c37c5e16b081950d389e3386 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 29 Aug 2017 14:08:59 -0400 Subject: Refactor TableOfContentsFilter's nested table of contents Most of the logic is now self-contained within the `HeaderNode` class. --- lib/banzai/filter/table_of_contents_filter.rb | 106 ++++++++++++++++---------- 1 file changed, 64 insertions(+), 42 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 8cb860c5a92..eda94bea5e0 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -15,7 +15,6 @@ module Banzai # `li` child elements. class TableOfContentsFilter < HTML::Pipeline::Filter PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u - HeaderNode = Struct.new(:level, :href, :text, :children, :parent) def call return doc if context[:no_header_anchors] @@ -23,56 +22,34 @@ module Banzai result[:toc] = "" headers = Hash.new(0) - - # root node of header-tree - header_root = HeaderNode.new(0, nil, nil, [], nil) - current_header = header_root + header_root = current_header = HeaderNode.new doc.css('h1, h2, h3, h4, h5, h6').each do |node| - text = node.text - - id = text.downcase - id.gsub!(PUNCTUATION_REGEXP, '') # remove punctuation - id.tr!(' ', '-') # replace spaces with dash - id.squeeze!('-') # replace multiple dashes with one - - uniq = (headers[id] > 0) ? "-#{headers[id]}" : '' - headers[id] += 1 - if header_content = node.children.first - # namespace detection will be automatically handled via javascript (see issue #22781) - namespace = "user-content-" + id = node + .text + .downcase + .gsub(PUNCTUATION_REGEXP, '') # remove punctuation + .tr(' ', '-') # replace spaces with dash + .squeeze('-') # replace multiple dashes with one + + uniq = headers[id] > 0 ? "-#{headers[id]}" : '' + headers[id] += 1 href = "#{id}#{uniq}" - level = node.name[1].to_i # get this header level - if level == current_header.level - # same as previous - parent = current_header.parent - elsif level > current_header.level - # larger (weaker) than previous - parent = current_header - else - # smaller (stronger) than previous - # search parent - parent = current_header - parent = parent.parent while parent.level >= level - end - - # create header-node and push as child - header_node = HeaderNode.new(level, href, text, [], parent) - parent.children.push(header_node) - current_header = header_node - - header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href)) + current_header = HeaderNode.new(node: node, href: href, previous_header: current_header) + + header_content.add_previous_sibling(anchor_tag(href)) end end - # extract header-tree if header_root.children.length > 0 - result[:toc] = %Q{
      \n} + result[:toc] = %q{
        } + header_root.children.each do |child| push_toc(child) end + result[:toc] << '
      ' end @@ -81,20 +58,65 @@ module Banzai private - def anchor_tag(id, href) - %Q{} + def anchor_tag(href) + %Q{} end def push_toc(header_node) result[:toc] << %Q{
    • #{header_node.text}} + if header_node.children.length > 0 result[:toc] << '
        ' + header_node.children.each do |child| push_toc(child) end + result[:toc] << '
      ' end - result[:toc] << '
      • \n' + + result[:toc] << '' + end + + class HeaderNode + attr_reader :node, :href, :parent, :children + + def initialize(node: nil, href: nil, previous_header: nil) + @node = node + @href = href + @children = [] + + find_parent(previous_header) + end + + def level + return 0 unless node + + @level ||= node.name[1].to_i + end + + def text + return '' unless node + + @text ||= node.text + end + + private + + def find_parent(previous_header) + return unless previous_header + + if level == previous_header.level + @parent = previous_header.parent + elsif level > previous_header.level + @parent = previous_header + else + @parent = previous_header + @parent = @parent.parent while @parent.level >= level + end + + @parent.children.push(self) + end end end end -- cgit v1.2.1 From fc27ca62dd3d8df86ce2e0f85668f561b0db43fe Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Fri, 1 Sep 2017 14:58:40 -0400 Subject: Refactor `HeaderNode#find_parent` Now the instance variable assignment is all contained within `#initialize`, and the `children.push` behavior is more explicit. --- lib/banzai/filter/table_of_contents_filter.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index eda94bea5e0..5522986ba21 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -86,7 +86,8 @@ module Banzai @href = href @children = [] - find_parent(previous_header) + @parent = find_parent(previous_header) + @parent.children.push(self) if @parent end def level @@ -107,15 +108,15 @@ module Banzai return unless previous_header if level == previous_header.level - @parent = previous_header.parent + parent = previous_header.parent elsif level > previous_header.level - @parent = previous_header + parent = previous_header else - @parent = previous_header - @parent = @parent.parent while @parent.level >= level + parent = previous_header + parent = parent.parent while parent.level >= level end - @parent.children.push(self) + parent end end end -- cgit v1.2.1 From 056158efbacf2f9d1b72402d56a0ebbe92cb71db Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Fri, 1 Sep 2017 15:04:32 -0400 Subject: Refactor table of contents building --- lib/banzai/filter/table_of_contents_filter.rb | 30 ++++++++++----------------- 1 file changed, 11 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 5522986ba21..47151626208 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -43,15 +43,7 @@ module Banzai end end - if header_root.children.length > 0 - result[:toc] = %q{
        } - - header_root.children.each do |child| - push_toc(child) - end - - result[:toc] << '
      ' - end + push_toc(header_root.children, root: true) doc end @@ -62,19 +54,19 @@ module Banzai %Q{} end - def push_toc(header_node) - result[:toc] << %Q{
    • #{header_node.text}} - - if header_node.children.length > 0 - result[:toc] << '
        ' + def push_toc(children, root: false) + return if children.empty? - header_node.children.each do |child| - push_toc(child) - end + klass = ' class="section-nav"' if root - result[:toc] << '
      ' - end + result[:toc] << "" + children.each { |child| push_anchor(child) } + result[:toc] << '
    ' + end + def push_anchor(header_node) + result[:toc] << %Q{
  • #{header_node.text}} + push_toc(header_node.children) result[:toc] << '
    • ' end -- cgit v1.2.1 From 6c49a628000605d1beb120431003abb329b9fd16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 17 Aug 2017 10:37:36 -0500 Subject: Restore some changes from !9199 --- lib/api/entities.rb | 2 +- lib/gitlab/themes.rb | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/themes.rb (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1d224d7bc21..d5f2c422c58 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -35,7 +35,7 @@ module API expose :confirmed_at expose :last_activity_on expose :email - expose :color_scheme_id, :projects_limit, :current_sign_in_at + expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb new file mode 100644 index 00000000000..19ab76ae80f --- /dev/null +++ b/lib/gitlab/themes.rb @@ -0,0 +1,87 @@ +module Gitlab + # Module containing GitLab's application theme definitions and helper methods + # for accessing them. + module Themes + extend self + + # Theme ID used when no `default_theme` configuration setting is provided. + APPLICATION_DEFAULT = 2 + + # Struct class representing a single Theme + Theme = Struct.new(:id, :name, :css_class) + + # All available Themes + THEMES = [ + Theme.new(1, 'Graphite', 'ui_graphite'), + Theme.new(2, 'Charcoal', 'ui_charcoal'), + Theme.new(3, 'Green', 'ui_green'), + Theme.new(4, 'Black', 'ui_black'), + Theme.new(5, 'Violet', 'ui_violet'), + Theme.new(6, 'Blue', 'ui_blue') + ].freeze + + # Convenience method to get a space-separated String of all the theme + # classes that might be applied to the `body` element + # + # Returns a String + def body_classes + THEMES.collect(&:css_class).uniq.join(' ') + end + + # Get a Theme by its ID + # + # If the ID is invalid, returns the default Theme. + # + # id - Integer ID + # + # Returns a Theme + def by_id(id) + THEMES.detect { |t| t.id == id } || default + end + + # Returns the number of defined Themes + def count + THEMES.size + end + + # Get the default Theme + # + # Returns a Theme + def default + by_id(default_id) + end + + # Iterate through each Theme + # + # Yields the Theme object + def each(&block) + THEMES.each(&block) + end + + # Get the Theme for the specified user, or the default + # + # user - User record + # + # Returns a Theme + def for_user(user) + if user + by_id(user.theme_id) + else + default + end + end + + private + + def default_id + id = Gitlab.config.gitlab.default_theme.to_i + + # Prevent an invalid configuration setting from causing an infinite loop + if id < THEMES.first.id || id > THEMES.last.id + APPLICATION_DEFAULT + else + id + end + end + end +end -- cgit v1.2.1 From 3c815f97d32d78abc0eea4f0dd2397e646cd9fd6 Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Thu, 31 Aug 2017 15:08:30 -0500 Subject: Add functionality for two themes --- lib/gitlab/themes.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 19ab76ae80f..1714b6d2c99 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -12,12 +12,8 @@ module Gitlab # All available Themes THEMES = [ - Theme.new(1, 'Graphite', 'ui_graphite'), - Theme.new(2, 'Charcoal', 'ui_charcoal'), - Theme.new(3, 'Green', 'ui_green'), - Theme.new(4, 'Black', 'ui_black'), - Theme.new(5, 'Violet', 'ui_violet'), - Theme.new(6, 'Blue', 'ui_blue') + Theme.new(1, 'Indigo', 'ui_indigo'), + Theme.new(2, 'Dark', 'ui_dark') ].freeze # Convenience method to get a space-separated String of all the theme -- cgit v1.2.1 From c3ac7311525bc23bd8b75addec34654aa87ebbae Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Tue, 5 Sep 2017 16:57:31 -0700 Subject: Add blue theme --- lib/gitlab/themes.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 1714b6d2c99..e0be799b551 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -13,7 +13,8 @@ module Gitlab # All available Themes THEMES = [ Theme.new(1, 'Indigo', 'ui_indigo'), - Theme.new(2, 'Dark', 'ui_dark') + Theme.new(2, 'Dark', 'ui_dark'), + Theme.new(3, 'Blue', 'ui_blue') ].freeze # Convenience method to get a space-separated String of all the theme -- cgit v1.2.1 From 7b567597eef25ca3af8af63a71bcc4dfefc2a694 Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Wed, 6 Sep 2017 08:09:54 -0700 Subject: Fix specs; start on light theme --- lib/gitlab/themes.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index e0be799b551..5e1e65854a3 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -14,7 +14,9 @@ module Gitlab THEMES = [ Theme.new(1, 'Indigo', 'ui_indigo'), Theme.new(2, 'Dark', 'ui_dark'), - Theme.new(3, 'Blue', 'ui_blue') + Theme.new(3, 'Light', 'ui_light'), + Theme.new(4, 'Blue', 'ui_blue'), + Theme.new(5, 'Green', 'ui_green'), ].freeze # Convenience method to get a space-separated String of all the theme -- cgit v1.2.1 From 6ee158197dc97924ab6f53ce6075cb3a8b9786ce Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Wed, 6 Sep 2017 11:37:40 -0700 Subject: Add previews in preferences --- lib/gitlab/themes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 5e1e65854a3..e871557e243 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -16,7 +16,7 @@ module Gitlab Theme.new(2, 'Dark', 'ui_dark'), Theme.new(3, 'Light', 'ui_light'), Theme.new(4, 'Blue', 'ui_blue'), - Theme.new(5, 'Green', 'ui_green'), + Theme.new(5, 'Green', 'ui_green') ].freeze # Convenience method to get a space-separated String of all the theme -- cgit v1.2.1 From c2e99b40f71ca7036cc07596aae164e92378263b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Wed, 6 Sep 2017 17:47:25 -0300 Subject: Implement fix for n+1 issue on `flatten_tree` helper --- lib/gitlab/git/tree.rb | 11 +++++++---- lib/gitlab/gitaly_client/commit_service.rb | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b54962a4456..5cf336af3c6 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -5,7 +5,7 @@ module Gitlab class Tree include Gitlab::EncodingHelper - attr_accessor :id, :root_id, :name, :path, :type, + attr_accessor :id, :root_id, :name, :path, :flat_path, :type, :mode, :commit_id, :submodule_url class << self @@ -19,8 +19,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| if is_enabled - client = Gitlab::GitalyClient::CommitService.new(repository) - client.tree_entries(repository, sha, path) + repository.gitaly_commit_client.tree_entries(repository, sha, path) else tree_entries_from_rugged(repository, sha, path) end @@ -88,7 +87,7 @@ module Gitlab end def initialize(options) - %w(id root_id name path type mode commit_id).each do |key| + %w(id root_id name path flat_path type mode commit_id).each do |key| self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end @@ -101,6 +100,10 @@ module Gitlab encode! @path end + def flat_path + encode! @flat_path + end + def dir? type == :tree end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 21a32a7e0db..0825a3a7694 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -88,14 +88,14 @@ module Gitlab response.flat_map do |message| message.entries.map do |gitaly_tree_entry| - entry_path = gitaly_tree_entry.path.dup Gitlab::Git::Tree.new( id: gitaly_tree_entry.oid, root_id: gitaly_tree_entry.root_oid, type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), - name: File.basename(entry_path), - path: entry_path, + name: File.basename(gitaly_tree_entry.path), + path: GitalyClient.encode(gitaly_tree_entry.path), + flat_path: GitalyClient.encode(gitaly_tree_entry.flat_path), commit_id: gitaly_tree_entry.commit_oid ) end -- cgit v1.2.1 From 2915bb27078a3eae0bac36bd2c3a2e1c4998130c Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Thu, 7 Sep 2017 09:21:52 +1100 Subject: Add API support for wiki pages --- lib/api/api.rb | 1 + lib/api/entities.rb | 10 ++++++ lib/api/helpers.rb | 6 ++++ lib/api/wikis.rb | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 lib/api/wikis.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 94df543853b..9f67cb598f6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -143,6 +143,7 @@ module API mount ::API::Variables mount ::API::GroupVariables mount ::API::Version + mount ::API::Wikis route :any, '*path' do error!('404 Not Found', 404) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e8dd61e493f..3c0fc4c2564 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,5 +1,15 @@ module API module Entities + class WikiPageBasic < Grape::Entity + expose :format + expose :slug + expose :title + end + + class WikiPage < WikiPageBasic + expose :content + end + class UserSafe < Grape::Entity expose :name, :username end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index b56fd2388b3..0932f8eb76f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -35,6 +35,12 @@ module API @project ||= find_project!(params[:id]) end + def wiki_page + page = user_project.wiki.find_page(params[:slug]) + + page || not_found!('Wiki Page') + end + def available_labels @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb new file mode 100644 index 00000000000..b3fc4e876ad --- /dev/null +++ b/lib/api/wikis.rb @@ -0,0 +1,89 @@ +module API + class Wikis < Grape::API + helpers do + params :wiki_page_params do + requires :content, type: String, desc: 'Content of a wiki page' + requires :title, type: String, desc: 'Title of a wiki page' + optional :format, + type: String, + values: ProjectWiki::MARKUPS.values.map(&:to_s), + default: 'markdown', + desc: 'Format of a wiki page. Available formats are markdown, rdoc, and asciidoc' + end + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Get a list of wiki pages' do + success Entities::WikiPageBasic + end + params do + optional :with_content, type: Boolean, default: false, desc: "Include pages' content" + end + get ':id/wikis' do + authorize! :read_wiki, user_project + + entity = params[:with_content] ? Entities::WikiPage : Entities::WikiPageBasic + present user_project.wiki.pages, with: entity + end + + desc 'Get a wiki page' do + success Entities::WikiPage + end + params do + requires :slug, type: String, desc: 'The slug of a wiki page' + end + get ':id/wikis/:slug' do + authorize! :read_wiki, user_project + + present wiki_page, with: Entities::WikiPage + end + + desc 'Create a wiki page' do + success Entities::WikiPage + end + params do + use :wiki_page_params + end + post ':id/wikis' do + authorize! :create_wiki, user_project + + page = WikiPages::CreateService.new(user_project, current_user, params).execute + + if page.valid? + present page, with: Entities::WikiPage + else + render_validation_error!(page) + end + end + + desc 'Update a wiki page' do + success Entities::WikiPage + end + params do + use :wiki_page_params + end + put ':id/wikis/:slug' do + authorize! :create_wiki, user_project + + page = WikiPages::UpdateService.new(user_project, current_user, params).execute(wiki_page) + + if page.valid? + present page, with: Entities::WikiPage + else + render_validation_error!(page) + end + end + + desc 'Delete a wiki page' + params do + requires :slug, type: String, desc: 'The slug of a wiki page' + end + delete ':id/wikis/:slug' do + authorize! :admin_wiki, user_project + + status 204 + WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page) + end + end + end +end -- cgit v1.2.1 From f7c8032e0993a6dc6bb808b0f2234324d3fe9707 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 6 Sep 2017 22:41:15 -0700 Subject: Add JSON logger in `log/api_json.log` for Grape API endpoints Closes #36189 --- lib/api/api.rb | 9 +++++++++ lib/gitlab/api_logger.rb | 8 ++++++++ 2 files changed, 17 insertions(+) create mode 100644 lib/gitlab/api_logger.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 1405a5d0f0e..63df22c508b 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -2,6 +2,15 @@ module API class API < Grape::API include APIGuard + LOG_FILENAME = Rails.root.join("log", "api_json.log") + + use GrapeLogging::Middleware::RequestLogger, + logger: ::Gitlab::ApiLogger.new(LOG_FILENAME), + formatter: GrapeLogging::Formatters::Json.new, + include: [ GrapeLogging::Loggers::Response.new, + GrapeLogging::Loggers::FilterParameters.new, + GrapeLogging::Loggers::ClientEnv.new ] + allow_access_with_scope :api prefix :api diff --git a/lib/gitlab/api_logger.rb b/lib/gitlab/api_logger.rb new file mode 100644 index 00000000000..09122b233ea --- /dev/null +++ b/lib/gitlab/api_logger.rb @@ -0,0 +1,8 @@ +module Gitlab + class ApiLogger < ::Logger + + def format_message(severity, timestamp, progname, message) + super + "\n" + end + end +end -- cgit v1.2.1 From cfd475a45ee2655fa0148b0b561f95b44fe8641b Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Tue, 15 Aug 2017 11:27:37 +0100 Subject: Removes default scope from sortable --- lib/api/broadcast_messages.rb | 2 +- lib/api/merge_request_diffs.rb | 2 +- lib/api/milestone_responses.rb | 2 +- lib/api/v3/merge_request_diffs.rb | 2 +- lib/api/v3/milestones.rb | 1 + lib/api/v3/projects.rb | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 0b45621ce7b..d7138b2f2fe 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -20,7 +20,7 @@ module API use :pagination end get do - messages = BroadcastMessage.all + messages = BroadcastMessage.all.order_id_desc present paginate(messages), with: Entities::BroadcastMessage end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index c3affcc6c6b..95ef8f42954 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -21,7 +21,7 @@ module API get ":id/merge_requests/:merge_request_iid/versions" do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present paginate(merge_request.merge_request_diffs), with: Entities::MergeRequestDiff + present paginate(merge_request.merge_request_diffs.order_id_desc), with: Entities::MergeRequestDiff end desc 'Get a single merge request diff version' do diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index ef09d9505d2..c570eace862 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -28,7 +28,7 @@ module API end def list_milestones_for(parent) - milestones = parent.milestones + milestones = parent.milestones.order_id_desc milestones = Milestone.filter_by_state(milestones, params[:state]) milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? milestones = filter_by_search(milestones, params[:search]) if params[:search] diff --git a/lib/api/v3/merge_request_diffs.rb b/lib/api/v3/merge_request_diffs.rb index 35f462e907b..22866fc2845 100644 --- a/lib/api/v3/merge_request_diffs.rb +++ b/lib/api/v3/merge_request_diffs.rb @@ -20,7 +20,7 @@ module API get ":id/merge_requests/:merge_request_id/versions" do merge_request = find_merge_request_with_access(params[:merge_request_id]) - present merge_request.merge_request_diffs, with: ::API::Entities::MergeRequestDiff + present merge_request.merge_request_diffs.order_id_desc, with: ::API::Entities::MergeRequestDiff end desc 'Get a single merge request diff version' do diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb index 4c7061d4939..9be4cf9d22a 100644 --- a/lib/api/v3/milestones.rb +++ b/lib/api/v3/milestones.rb @@ -34,6 +34,7 @@ module API milestones = user_project.milestones milestones = filter_milestones_state(milestones, params[:state]) milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + milestones = milestones.order_id_desc present paginate(milestones), with: ::API::Entities::Milestone end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 449876c10d9..6b338ce2d6a 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -119,7 +119,7 @@ module API get do authenticate! - present_projects current_user.authorized_projects, + present_projects current_user.authorized_projects.order_id_desc, with: ::API::V3::Entities::ProjectWithAccess end -- cgit v1.2.1 From 99dcf870dd46299ffec2e5067c9b3223de3315ce Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Thu, 7 Sep 2017 06:43:12 -0700 Subject: Use indigo as default theme --- lib/gitlab/themes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index e871557e243..37054d98102 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -5,7 +5,7 @@ module Gitlab extend self # Theme ID used when no `default_theme` configuration setting is provided. - APPLICATION_DEFAULT = 2 + APPLICATION_DEFAULT = 1 # Struct class representing a single Theme Theme = Struct.new(:id, :name, :css_class) -- cgit v1.2.1 From c304dfd4d6ca0f52537044742bb6dd6c219bdbbf Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 7 Sep 2017 07:02:46 -0700 Subject: Fix Rubocop failures in API logger --- lib/api/api.rb | 8 +++++--- lib/gitlab/api_logger.rb | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 63df22c508b..0ab0c8c490a 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -7,9 +7,11 @@ module API use GrapeLogging::Middleware::RequestLogger, logger: ::Gitlab::ApiLogger.new(LOG_FILENAME), formatter: GrapeLogging::Formatters::Json.new, - include: [ GrapeLogging::Loggers::Response.new, - GrapeLogging::Loggers::FilterParameters.new, - GrapeLogging::Loggers::ClientEnv.new ] + include: [ + GrapeLogging::Loggers::Response.new, + GrapeLogging::Loggers::FilterParameters.new, + GrapeLogging::Loggers::ClientEnv.new + ] allow_access_with_scope :api prefix :api diff --git a/lib/gitlab/api_logger.rb b/lib/gitlab/api_logger.rb index 09122b233ea..dcb194a1b89 100644 --- a/lib/gitlab/api_logger.rb +++ b/lib/gitlab/api_logger.rb @@ -1,6 +1,5 @@ module Gitlab class ApiLogger < ::Logger - def format_message(severity, timestamp, progname, message) super + "\n" end -- cgit v1.2.1 From 26264625cbc2692bc113c1e701cb15eecbd010d2 Mon Sep 17 00:00:00 2001 From: Ruben Davila Date: Thu, 7 Sep 2017 09:10:39 -0500 Subject: Small refactor after last code review. --- lib/gitlab/themes.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 37054d98102..d43eff5ba4a 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -73,13 +73,11 @@ module Gitlab private def default_id - id = Gitlab.config.gitlab.default_theme.to_i + @default_id ||= begin + id = Gitlab.config.gitlab.default_theme.to_i + theme_ids = THEMES.map(&:id) - # Prevent an invalid configuration setting from causing an infinite loop - if id < THEMES.first.id || id > THEMES.last.id - APPLICATION_DEFAULT - else - id + theme_ids.include?(id) ? id : APPLICATION_DEFAULT end end end -- cgit v1.2.1 From 35dec2c3e87f2f44c3ab0269e7f737afdc28801a Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 7 Sep 2017 07:48:13 -0700 Subject: Use a custom GrapeLogging formatter to get the timestamp --- lib/api/api.rb | 4 ++-- lib/gitlab/api_logger.rb | 7 ------- .../formatters/lograge_with_timestamp.rb | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 9 deletions(-) delete mode 100644 lib/gitlab/api_logger.rb create mode 100644 lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 0ab0c8c490a..af0b38963f5 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -5,8 +5,8 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") use GrapeLogging::Middleware::RequestLogger, - logger: ::Gitlab::ApiLogger.new(LOG_FILENAME), - formatter: GrapeLogging::Formatters::Json.new, + logger: Logger.new(LOG_FILENAME), + formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ GrapeLogging::Loggers::Response.new, GrapeLogging::Loggers::FilterParameters.new, diff --git a/lib/gitlab/api_logger.rb b/lib/gitlab/api_logger.rb deleted file mode 100644 index dcb194a1b89..00000000000 --- a/lib/gitlab/api_logger.rb +++ /dev/null @@ -1,7 +0,0 @@ -module Gitlab - class ApiLogger < ::Logger - def format_message(severity, timestamp, progname, message) - super + "\n" - end - end -end diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb new file mode 100644 index 00000000000..1e1fdabca93 --- /dev/null +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -0,0 +1,19 @@ +module Gitlab + module GrapeLogging + module Formatters + class LogrageWithTimestamp + def call(severity, datetime, _, data) + time = data.delete :time + attributes = { + time: datetime.utc.iso8601(3), + severity: severity, + duration: time[:total], + db: time[:db], + view: time[:view] + }.merge(data) + ::Lograge.formatter.call(attributes) + "\n" + end + end + end + end +end -- cgit v1.2.1 From 94680e1448f9e3af3dfc43e33a74609cdc0ecb69 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Thu, 7 Sep 2017 17:39:00 +0000 Subject: Gitaly feature toggles are on by default in development environments --- lib/gitlab/gitaly_client.rb | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 9a5f4f598b2..a3dc2cd0b60 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -70,21 +70,41 @@ module Gitlab params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end - def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) + # Evaluates whether a feature toggle is on or off + def self.feature_enabled?(feature_name, status: MigrationStatus::OPT_IN) + # Disabled features are always off! return false if status == MigrationStatus::DISABLED - feature = Feature.get("gitaly_#{feature}") + feature = Feature.get("gitaly_#{feature_name}") - # If the feature hasn't been set, turn it on if it's opt-out - return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature) + # If the feature has been set, always evaluate + if Feature.persisted?(feature) + if feature.percentage_of_time_value > 0 + # Probabilistically enable this feature + return Random.rand() * 100 < feature.percentage_of_time_value + end + + return feature.enabled? + end - if feature.percentage_of_time_value > 0 - # Probabilistically enable this feature - return Random.rand() * 100 < feature.percentage_of_time_value + # If the feature has not been set, the default depends + # on it's status + case status + when MigrationStatus::OPT_OUT + true + when MigrationStatus::OPT_IN + opt_into_all_features? + else + false end + end - feature.enabled? + # opt_into_all_features? returns true when the current environment + # is one in which we opt into features automatically + def self.opt_into_all_features? + Rails.env.development? || ENV["GITALY_FEATURE_DEFAULT_ON"] == "1" end + private_class_method :opt_into_all_features? def self.migrate(feature, status: MigrationStatus::OPT_IN) is_enabled = feature_enabled?(feature, status: status) -- cgit v1.2.1 From 39298575a819ade6ad4f9e37a7f22592a05d21f8 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Mon, 4 Sep 2017 18:55:04 +0100 Subject: Adds exclusive lease to Git garbage collect worker. --- lib/gitlab/exclusive_lease.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 3784f6c4947..3f7b42456af 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -25,6 +25,12 @@ module Gitlab end EOS + def self.get_uuid(key) + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_shared_state_key(key)) || false + end + end + def self.cancel(key, uuid) Gitlab::Redis::SharedState.with do |redis| redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) @@ -35,10 +41,10 @@ module Gitlab "gitlab:exclusive_lease:#{key}" end - def initialize(key, timeout:) + def initialize(key, uuid: nil, timeout:) @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout - @uuid = SecureRandom.uuid + @uuid = uuid || SecureRandom.uuid end # Try to obtain the lease. Return lease UUID on success, -- cgit v1.2.1 From 62bb6235c229a869052180f9709c4801116f02cc Mon Sep 17 00:00:00 2001 From: Ruben Davila Date: Thu, 7 Sep 2017 13:35:45 -0500 Subject: Make Members with Owner and Master roles always able to create subgroups --- lib/api/groups.rb | 7 ++++++- lib/api/helpers.rb | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 31a918eda60..e817dcbbc4b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -74,7 +74,12 @@ module API use :optional_params end post do - authorize! :create_group + parent_group = find_group!(params[:parent_id]) if params[:parent_id].present? + if parent_group + authorize! :create_subgroup, parent_group + else + authorize! :create_group + end group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8b03df65ae4..00dbc2aee7a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -93,7 +93,7 @@ module API end def find_group(id) - if id =~ /^\d+$/ + if id.to_s =~ /^\d+$/ Group.find_by(id: id) else Group.find_by_full_path(id) -- cgit v1.2.1