From 9ad48a7438048ca19ff847e5fa7f2ccd7c6d59af Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 9 Oct 2017 13:07:55 +0200 Subject: Extract class responsible for building a pipeline --- lib/gitlab/ci/pipeline/chain/build.rb | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 lib/gitlab/ci/pipeline/chain/build.rb (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb new file mode 100644 index 00000000000..efd1da733c5 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -0,0 +1,56 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Build < Chain::Base + include Chain::Helpers + + def perform! + @pipeline.assign_attributes( + source: @command.source, + project: @project, + ref: ref, + sha: sha, + before_sha: before_sha, + tag: tag_exists?, + trigger_requests: Array(@command.trigger_request), + user: @current_user, + pipeline_schedule: @command.schedule, + protected: protected_ref? + ) + end + + def break? + false + end + + private + + def ref + @ref ||= Gitlab::Git.ref_name(origin_ref) + end + + def sha + @project.commit(origin_sha || origin_ref).try(:id) + end + + def origin_ref + @command.origin_ref + end + + def origin_sha + @command.checkout_sha || @command.after_sha + end + + def before_sha + @command.checkout_sha || @command.before_sha || Gitlab::Git::BLANK_SHA + end + + def protected_ref? + @project.protected_for?(ref) + end + end + end + end + end +end -- cgit v1.2.1 From eedf43d69bb2d3e5ed73b751565beb5d1badd8c7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 11 Oct 2017 14:48:28 +0200 Subject: Set pipeline config source attribute in a build step --- lib/gitlab/ci/pipeline/chain/build.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index efd1da733c5..a126dded1ae 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -18,6 +18,8 @@ module Gitlab pipeline_schedule: @command.schedule, protected: protected_ref? ) + + @pipeline.set_config_source end def break? -- cgit v1.2.1 From 9b58b8e363fd388635385085c58be3d4637eaa45 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 6 Nov 2017 22:20:44 +0900 Subject: Do not allow jobs to be erased --- lib/api/jobs.rb | 2 +- lib/api/v3/builds.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 3c1c412ba42..a116ab3c9bd 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -136,7 +136,7 @@ module API authorize_update_builds! build = find_build!(params[:job_id]) - authorize!(:update_build, build) + authorize!(:erase_build, build) return forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index f493fd7c7ec..fa0bef39602 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -169,7 +169,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) - authorize!(:update_build, build) + authorize!(:erase_build, build) return forbidden!('Build is not erasable!') unless build.erasable? build.erase(erased_by: current_user) -- cgit v1.2.1 From c8eb2a914b6f9348ffa16436853964998c115085 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 6 Nov 2017 23:24:25 +0900 Subject: Fix spec. Revert update check. --- lib/api/jobs.rb | 1 + lib/api/v3/builds.rb | 1 + 2 files changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index a116ab3c9bd..6dcbe2ff936 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -136,6 +136,7 @@ module API authorize_update_builds! build = find_build!(params[:job_id]) + authorize!(:update_build, build) authorize!(:erase_build, build) return forbidden!('Job is not erasable!') unless build.erasable? diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index fa0bef39602..1c0f9f73c78 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -169,6 +169,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) authorize!(:erase_build, build) return forbidden!('Build is not erasable!') unless build.erasable? -- cgit v1.2.1 From afef38533727cf32a7be324243a25b4db5eb5498 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 7 Nov 2017 02:47:05 +0900 Subject: Add doc. Fix spec. Add erase_build in protected_ref rule --- lib/api/jobs.rb | 1 - lib/api/v3/builds.rb | 1 - 2 files changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 6dcbe2ff936..a116ab3c9bd 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -136,7 +136,6 @@ module API authorize_update_builds! build = find_build!(params[:job_id]) - authorize!(:update_build, build) authorize!(:erase_build, build) return forbidden!('Job is not erasable!') unless build.erasable? diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 1c0f9f73c78..fa0bef39602 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -169,7 +169,6 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) - authorize!(:update_build, build) authorize!(:erase_build, build) return forbidden!('Build is not erasable!') unless build.erasable? -- cgit v1.2.1 From f4ed780ef5fc76b7704742d4886ac435c3e5ab98 Mon Sep 17 00:00:00 2001 From: George Andrinopoulos Date: Mon, 6 Nov 2017 19:51:24 +0200 Subject: Add Group Milestone sorting --- lib/milestone_array.rb | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 lib/milestone_array.rb (limited to 'lib') diff --git a/lib/milestone_array.rb b/lib/milestone_array.rb new file mode 100644 index 00000000000..9b3f2acc123 --- /dev/null +++ b/lib/milestone_array.rb @@ -0,0 +1,40 @@ +module MilestoneArray + class << self + def sort(array, sort_method) + case sort_method + when 'due_date_asc' + sort_asc_nulls_last(array, 'due_date') + when 'due_date_desc' + sort_desc_nulls_last(array, 'due_date') + when 'start_date_asc' + sort_asc_nulls_last(array, 'start_date') + when 'start_date_desc' + sort_desc_nulls_last(array, 'start_date') + when 'name_asc' + sort_asc(array, 'title') + when 'name_desc' + sort_desc(array, 'title') + else + array + end + end + + private + + def sort_asc_nulls_last(array, attribute) + array.select(&attribute.to_sym).sort_by(&attribute.to_sym) + array.reject(&attribute.to_sym) + end + + def sort_desc_nulls_last(array, attribute) + array.select(&attribute.to_sym).sort_by(&attribute.to_sym).reverse + array.reject(&attribute.to_sym) + end + + def sort_asc(array, attribute) + array.sort_by(&attribute.to_sym) + end + + def sort_desc(array, attribute) + array.sort_by(&attribute.to_sym).reverse + end + end +end -- cgit v1.2.1 From 4d4ddb6004e6f7f56b337a49c6eedaad70d70862 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Tue, 7 Nov 2017 14:00:21 +0000 Subject: Fail when issuable_meta_data is called on an unlimited collection This method can be called with an array, or a relation: 1. Arrays always have a limited amount of values, so that's fine. 2. If the relation does not have a limit value applied, then we will load every single object in that collection, and prevent N+1 queries for the metadata for that. But that's wrong, because we should never call this without an explicit limit set. So we raise in that case, and this commit will see which specs fail. The only failing specs here were the issues API specs, and the specs for IssuableMetadata itself, and both have been addressed. --- lib/api/issues.rb | 12 ++++++------ lib/gitlab/issuable_metadata.rb | 8 ++++++++ 2 files changed, 14 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0df41dcc903..74dfd9f96de 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -68,7 +68,7 @@ module API desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' end get do - issues = find_issues + issues = paginate(find_issues) options = { with: Entities::IssueBasic, @@ -76,7 +76,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end end @@ -95,7 +95,7 @@ module API get ":id/issues" do group = find_group!(params[:id]) - issues = find_issues(group_id: group.id) + issues = paginate(find_issues(group_id: group.id)) options = { with: Entities::IssueBasic, @@ -103,7 +103,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end end @@ -124,7 +124,7 @@ module API get ":id/issues" do project = find_project!(params[:id]) - issues = find_issues(project_id: project.id) + issues = paginate(find_issues(project_id: project.id)) options = { with: Entities::IssueBasic, @@ -133,7 +133,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end desc 'Get a single project issue' do diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index 977c05910d3..0c9de72329c 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -1,6 +1,14 @@ module Gitlab module IssuableMetadata def issuable_meta_data(issuable_collection, collection_type) + # ActiveRecord uses Object#extend for null relations. + if !(issuable_collection.singleton_class < ActiveRecord::NullRelation) && + issuable_collection.respond_to?(:limit_value) && + issuable_collection.limit_value.nil? + + raise 'Collection must have a limit applied for preloading meta-data' + end + # map has to be used here since using pluck or select will # throw an error when ordering issuables by priority which inserts # a new order into the collection. -- cgit v1.2.1 From 4dea7944c46287707b6b65ca10e0af0b69a57a21 Mon Sep 17 00:00:00 2001 From: Joe Marty Date: Tue, 7 Nov 2017 11:42:25 -0600 Subject: Updates tests to reflect sign_out route change - Also remove sign_out DELETE route from read-only whitelist routes --- lib/gitlab/middleware/read_only.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 8853dfa3d2d..5e4932e4e57 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -66,11 +66,7 @@ module Gitlab end def whitelisted_routes - logout_route || grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def logout_route - route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy' + grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end def sidekiq_route -- cgit v1.2.1 From a4d71cba7ef80e6f3c10f148dd1edfbef7f82893 Mon Sep 17 00:00:00 2001 From: George Andrinopoulos Date: Tue, 7 Nov 2017 20:57:30 +0200 Subject: Add group milestone to feature spec and minor changes --- lib/milestone_array.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/milestone_array.rb b/lib/milestone_array.rb index 9b3f2acc123..4ed8485b36a 100644 --- a/lib/milestone_array.rb +++ b/lib/milestone_array.rb @@ -13,7 +13,7 @@ module MilestoneArray when 'name_asc' sort_asc(array, 'title') when 'name_desc' - sort_desc(array, 'title') + sort_asc(array, 'title').reverse else array end @@ -22,19 +22,19 @@ module MilestoneArray private def sort_asc_nulls_last(array, attribute) - array.select(&attribute.to_sym).sort_by(&attribute.to_sym) + array.reject(&attribute.to_sym) + attribute = attribute.to_sym + + array.select(&attribute).sort_by(&attribute) + array.reject(&attribute) end def sort_desc_nulls_last(array, attribute) - array.select(&attribute.to_sym).sort_by(&attribute.to_sym).reverse + array.reject(&attribute.to_sym) + attribute = attribute.to_sym + + array.select(&attribute).sort_by(&attribute).reverse + array.reject(&attribute) end def sort_asc(array, attribute) array.sort_by(&attribute.to_sym) end - - def sort_desc(array, attribute) - array.sort_by(&attribute.to_sym).reverse - end end end -- cgit v1.2.1 From bda30182e0d6c0be9f89e2e9a4e8b8325ad7ccb7 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Sun, 22 Oct 2017 01:55:59 +0300 Subject: Add returning IDs to Gitlab::Database.bulk_insert This adds the keyword argument "return_ids" to Gitlab::Database.bulk_insert. When set to `true` (and PostgreSQL is used) this method will return an Array of the IDs of the inserted rows, otherwise it will return an empty Array. --- lib/gitlab/database.rb | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 43a00d6cedb..cd7b4c043da 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -108,20 +108,41 @@ module Gitlab end end - def self.bulk_insert(table, rows) + # Bulk inserts a number of rows into a table, optionally returning their + # IDs. + # + # table - The name of the table to insert the rows into. + # rows - An Array of Hash instances, each mapping the columns to their + # values. + # return_ids - When set to true the return value will be an Array of IDs of + # the inserted rows, this only works on PostgreSQL. + def self.bulk_insert(table, rows, return_ids: false) return if rows.empty? keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } + return_ids = false if mysql? tuples = rows.map do |row| row.values_at(*keys).map { |value| connection.quote(value) } end - connection.execute <<-EOF + sql = <<-EOF INSERT INTO #{table} (#{columns.join(', ')}) VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} EOF + + if return_ids + sql << 'RETURNING id' + end + + result = connection.execute(sql) + + if return_ids + result.values.map { |tuple| tuple[0].to_i } + else + [] + end end def self.sanitize_timestamp(timestamp) -- cgit v1.2.1 From 90be53c5d39bff5e371cf8a6e11a39bf5dad7bcc Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 30 Oct 2017 16:10:31 +0100 Subject: Cache feature names in RequestStore The GitHub importer (and probably other parts of our code) ends up calling Feature.persisted? many times (via Gitaly). By storing this data in RequestStore we can save ourselves _a lot_ of database queries. Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/39361 --- lib/feature.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/feature.rb b/lib/feature.rb index 4bd29aed687..ac3bc65c0d5 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -5,6 +5,10 @@ class Feature class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature # Using `self.table_name` won't work. ActiveRecord bug? superclass.table_name = 'features' + + def self.feature_names + pluck(:key) + end end class FlipperGate < Flipper::Adapters::ActiveRecord::Gate @@ -22,11 +26,19 @@ class Feature flipper.feature(key) end + def persisted_names + if RequestStore.active? + RequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names + else + FlipperFeature.feature_names + end + end + def persisted?(feature) # Flipper creates on-memory features when asked for a not-yet-created one. # If we want to check if a feature has been actually set, we look for it # on the persisted features list. - all.map(&:name).include?(feature.name) + persisted_names.include?(feature.name) end def enabled?(key, thing = nil) -- cgit v1.2.1 From 4dfe26cd8b6863b7e6c81f5c280cdafe9b6e17b6 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 13 Oct 2017 18:50:36 +0200 Subject: Rewrite the GitHub importer from scratch Prior to this MR there were two GitHub related importers: * Github::Import: the main importer used for GitHub projects * Gitlab::GithubImport: importer that's somewhat confusingly used for importing Gitea projects (apparently they have a compatible API) This MR renames the Gitea importer to Gitlab::LegacyGithubImport and introduces a new GitHub importer in the Gitlab::GithubImport namespace. This new GitHub importer uses Sidekiq for importing multiple resources in parallel, though it also has the ability to import data sequentially should this be necessary. The new code is spread across the following directories: * lib/gitlab/github_import: this directory contains most of the importer code such as the classes used for importing resources. * app/workers/gitlab/github_import: this directory contains the Sidekiq workers, most of which simply use the code from the directory above. * app/workers/concerns/gitlab/github_import: this directory provides a few modules that are included in every GitHub importer worker. == Stages The import work is divided into separate stages, with each stage importing a specific set of data. Stages will schedule the work that needs to be performed, followed by scheduling a job for the "AdvanceStageWorker" worker. This worker will periodically check if all work is completed and schedule the next stage if this is the case. If work is not yet completed this worker will reschedule itself. Using this approach we don't have to block threads by calling `sleep()`, as doing so for large projects could block the thread from doing any work for many hours. == Retrying Work Workers will reschedule themselves whenever necessary. For example, hitting the GitHub API's rate limit will result in jobs rescheduling themselves. These jobs are not processed until the rate limit has been reset. == User Lookups Part of the importing process involves looking up user details in the GitHub API so we can map them to GitLab users. The old importer used an in-memory cache, but this obviously doesn't work when the work is spread across different threads. The new importer uses a Redis cache and makes sure we only perform API/database calls if absolutely necessary. Frequently used keys are refreshed, and lookup misses are also cached; removing the need for performing API/database calls if we know we don't have the data we're looking for. == Performance & Models The new importer in various places uses raw INSERT statements (as generated by `Gitlab::Database.bulk_insert`) instead of using Rails models. This allows us to bypass any validations and callbacks, drastically reducing the number of SQL queries and Gitaly RPC calls necessary to import projects. To ensure the code produces valid data the corresponding tests check if the produced rows are valid according to the model validation rules. --- lib/gitlab/git/repository.rb | 5 + lib/gitlab/github_import.rb | 34 +++ lib/gitlab/github_import/base_formatter.rb | 26 -- lib/gitlab/github_import/branch_formatter.rb | 37 --- lib/gitlab/github_import/bulk_importing.rb | 25 ++ lib/gitlab/github_import/caching.rb | 151 ++++++++++ lib/gitlab/github_import/client.rb | 239 ++++++++------- lib/gitlab/github_import/comment_formatter.rb | 69 ----- lib/gitlab/github_import/importer.rb | 329 --------------------- .../github_import/importer/diff_note_importer.rb | 63 ++++ .../github_import/importer/diff_notes_importer.rb | 31 ++ .../importer/issue_and_label_links_importer.rb | 25 ++ .../github_import/importer/issue_importer.rb | 81 +++++ .../github_import/importer/issues_importer.rb | 35 +++ .../github_import/importer/label_links_importer.rb | 52 ++++ .../github_import/importer/labels_importer.rb | 55 ++++ .../github_import/importer/milestones_importer.rb | 58 ++++ lib/gitlab/github_import/importer/note_importer.rb | 54 ++++ .../github_import/importer/notes_importer.rb | 31 ++ .../importer/pull_request_importer.rb | 91 ++++++ .../importer/pull_requests_importer.rb | 83 ++++++ .../github_import/importer/releases_importer.rb | 55 ++++ .../github_import/importer/repository_importer.rb | 96 ++++++ lib/gitlab/github_import/issuable_finder.rb | 81 +++++ lib/gitlab/github_import/issuable_formatter.rb | 66 ----- lib/gitlab/github_import/issue_formatter.rb | 32 -- lib/gitlab/github_import/label_finder.rb | 37 +++ lib/gitlab/github_import/label_formatter.rb | 37 --- lib/gitlab/github_import/markdown_text.rb | 30 ++ lib/gitlab/github_import/milestone_finder.rb | 40 +++ lib/gitlab/github_import/milestone_formatter.rb | 40 --- lib/gitlab/github_import/page_counter.rb | 31 ++ lib/gitlab/github_import/parallel_importer.rb | 44 +++ lib/gitlab/github_import/parallel_scheduling.rb | 162 ++++++++++ lib/gitlab/github_import/project_creator.rb | 52 ---- lib/gitlab/github_import/pull_request_formatter.rb | 90 ------ lib/gitlab/github_import/rate_limit_error.rb | 9 + lib/gitlab/github_import/release_formatter.rb | 27 -- lib/gitlab/github_import/representation.rb | 25 ++ .../github_import/representation/diff_note.rb | 87 ++++++ .../representation/expose_attribute.rb | 26 ++ lib/gitlab/github_import/representation/issue.rb | 80 +++++ lib/gitlab/github_import/representation/note.rb | 70 +++++ .../github_import/representation/pull_request.rb | 114 +++++++ lib/gitlab/github_import/representation/to_hash.rb | 31 ++ lib/gitlab/github_import/representation/user.rb | 34 +++ lib/gitlab/github_import/sequential_importer.rb | 50 ++++ lib/gitlab/github_import/user_finder.rb | 164 ++++++++++ lib/gitlab/github_import/user_formatter.rb | 45 --- lib/gitlab/github_import/wiki_formatter.rb | 19 -- lib/gitlab/import_sources.rb | 2 +- lib/gitlab/job_waiter.rb | 8 +- lib/gitlab/legacy_github_import/base_formatter.rb | 26 ++ .../legacy_github_import/branch_formatter.rb | 37 +++ lib/gitlab/legacy_github_import/client.rb | 148 +++++++++ .../legacy_github_import/comment_formatter.rb | 69 +++++ lib/gitlab/legacy_github_import/importer.rb | 329 +++++++++++++++++++++ .../legacy_github_import/issuable_formatter.rb | 66 +++++ lib/gitlab/legacy_github_import/issue_formatter.rb | 32 ++ lib/gitlab/legacy_github_import/label_formatter.rb | 37 +++ .../legacy_github_import/milestone_formatter.rb | 40 +++ lib/gitlab/legacy_github_import/project_creator.rb | 52 ++++ .../legacy_github_import/pull_request_formatter.rb | 90 ++++++ .../legacy_github_import/release_formatter.rb | 27 ++ lib/gitlab/legacy_github_import/user_formatter.rb | 45 +++ lib/gitlab/legacy_github_import/wiki_formatter.rb | 19 ++ 66 files changed, 3301 insertions(+), 974 deletions(-) create mode 100644 lib/gitlab/github_import.rb delete mode 100644 lib/gitlab/github_import/base_formatter.rb delete mode 100644 lib/gitlab/github_import/branch_formatter.rb create mode 100644 lib/gitlab/github_import/bulk_importing.rb create mode 100644 lib/gitlab/github_import/caching.rb delete mode 100644 lib/gitlab/github_import/comment_formatter.rb delete mode 100644 lib/gitlab/github_import/importer.rb create mode 100644 lib/gitlab/github_import/importer/diff_note_importer.rb create mode 100644 lib/gitlab/github_import/importer/diff_notes_importer.rb create mode 100644 lib/gitlab/github_import/importer/issue_and_label_links_importer.rb create mode 100644 lib/gitlab/github_import/importer/issue_importer.rb create mode 100644 lib/gitlab/github_import/importer/issues_importer.rb create mode 100644 lib/gitlab/github_import/importer/label_links_importer.rb create mode 100644 lib/gitlab/github_import/importer/labels_importer.rb create mode 100644 lib/gitlab/github_import/importer/milestones_importer.rb create mode 100644 lib/gitlab/github_import/importer/note_importer.rb create mode 100644 lib/gitlab/github_import/importer/notes_importer.rb create mode 100644 lib/gitlab/github_import/importer/pull_request_importer.rb create mode 100644 lib/gitlab/github_import/importer/pull_requests_importer.rb create mode 100644 lib/gitlab/github_import/importer/releases_importer.rb create mode 100644 lib/gitlab/github_import/importer/repository_importer.rb create mode 100644 lib/gitlab/github_import/issuable_finder.rb delete mode 100644 lib/gitlab/github_import/issuable_formatter.rb delete mode 100644 lib/gitlab/github_import/issue_formatter.rb create mode 100644 lib/gitlab/github_import/label_finder.rb delete mode 100644 lib/gitlab/github_import/label_formatter.rb create mode 100644 lib/gitlab/github_import/markdown_text.rb create mode 100644 lib/gitlab/github_import/milestone_finder.rb delete mode 100644 lib/gitlab/github_import/milestone_formatter.rb create mode 100644 lib/gitlab/github_import/page_counter.rb create mode 100644 lib/gitlab/github_import/parallel_importer.rb create mode 100644 lib/gitlab/github_import/parallel_scheduling.rb delete mode 100644 lib/gitlab/github_import/project_creator.rb delete mode 100644 lib/gitlab/github_import/pull_request_formatter.rb create mode 100644 lib/gitlab/github_import/rate_limit_error.rb delete mode 100644 lib/gitlab/github_import/release_formatter.rb create mode 100644 lib/gitlab/github_import/representation.rb create mode 100644 lib/gitlab/github_import/representation/diff_note.rb create mode 100644 lib/gitlab/github_import/representation/expose_attribute.rb create mode 100644 lib/gitlab/github_import/representation/issue.rb create mode 100644 lib/gitlab/github_import/representation/note.rb create mode 100644 lib/gitlab/github_import/representation/pull_request.rb create mode 100644 lib/gitlab/github_import/representation/to_hash.rb create mode 100644 lib/gitlab/github_import/representation/user.rb create mode 100644 lib/gitlab/github_import/sequential_importer.rb create mode 100644 lib/gitlab/github_import/user_finder.rb delete mode 100644 lib/gitlab/github_import/user_formatter.rb delete mode 100644 lib/gitlab/github_import/wiki_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/base_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/branch_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/client.rb create mode 100644 lib/gitlab/legacy_github_import/comment_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/importer.rb create mode 100644 lib/gitlab/legacy_github_import/issuable_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/issue_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/label_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/milestone_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/project_creator.rb create mode 100644 lib/gitlab/legacy_github_import/pull_request_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/release_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/user_formatter.rb create mode 100644 lib/gitlab/legacy_github_import/wiki_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index df4ad586e12..d236e1b03e6 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -920,6 +920,11 @@ module Gitlab false end + # Returns true if a remote exists. + def remote_exists?(name) + rugged.remotes[name].present? + end + # Update the specified remote using the values in the +options+ hash # # Example diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb new file mode 100644 index 00000000000..d2ae4c1255e --- /dev/null +++ b/lib/gitlab/github_import.rb @@ -0,0 +1,34 @@ +module Gitlab + module GithubImport + def self.new_client_for(project, token: nil, parallel: true) + token_to_use = token || project.import_data&.credentials&.fetch(:user) + + Client.new(token_to_use, parallel: parallel) + end + + # Inserts a raw row and returns the ID of the inserted row. + # + # attributes - The attributes/columns to set. + # relation - An ActiveRecord::Relation to use for finding the ID of the row + # when using MySQL. + def self.insert_and_return_id(attributes, relation) + # We use bulk_insert here so we can bypass any queries executed by + # callbacks or validation rules, as doing this wouldn't scale when + # importing very large projects. + result = Gitlab::Database + .bulk_insert(relation.table_name, [attributes], return_ids: true) + + # MySQL doesn't support returning the IDs of a bulk insert in a way that + # is not a pain, so in this case we'll issue an extra query instead. + result.first || + relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + end + + # Returns the ID of the ghost user. + def self.ghost_user_id + key = 'github-import/ghost-user-id' + + Caching.read_integer(key) || Caching.write(key, User.select(:id).ghost.id) + end + end +end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb deleted file mode 100644 index f330041cc00..00000000000 --- a/lib/gitlab/github_import/base_formatter.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Gitlab - module GithubImport - class BaseFormatter - attr_reader :client, :formatter, :project, :raw_data - - def initialize(project, raw_data, client = nil) - @project = project - @raw_data = raw_data - @client = client - @formatter = Gitlab::ImportFormatter.new - end - - def create! - association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend - - association.find_or_create_by!(find_condition) do |record| - record.attributes = attributes - end - end - - def url - raw_data.url || '' - end - end - end -end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb deleted file mode 100644 index 8aa885fb811..00000000000 --- a/lib/gitlab/github_import/branch_formatter.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module GithubImport - class BranchFormatter < BaseFormatter - delegate :repo, :sha, :ref, to: :raw_data - - def exists? - branch_exists? && commit_exists? - end - - def valid? - sha.present? && ref.present? - end - - def user - raw_data.user&.login || 'unknown' - end - - def short_sha - Commit.truncate_sha(sha) - end - - private - - def branch_exists? - project.repository.branch_exists?(ref) - end - - def commit_exists? - project.repository.branch_names_contains(sha).include?(ref) - end - - def short_id - sha.to_s[0..7] - end - end - end -end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb new file mode 100644 index 00000000000..147597289cf --- /dev/null +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module BulkImporting + # Builds and returns an Array of objects to bulk insert into the + # database. + # + # enum - An Enumerable that returns the objects to turn into database + # rows. + def build_database_rows(enum) + enum.each_with_object([]) do |(object, _), rows| + rows << build(object) unless already_imported?(object) + end + end + + # Bulk inserts the given rows into the database. + def bulk_insert(model, rows, batch_size: 100) + rows.each_slice(batch_size) do |slice| + Gitlab::Database.bulk_insert(model.table_name, slice) + end + end + end + end +end diff --git a/lib/gitlab/github_import/caching.rb b/lib/gitlab/github_import/caching.rb new file mode 100644 index 00000000000..b08f133794f --- /dev/null +++ b/lib/gitlab/github_import/caching.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Caching + # The default timeout of the cache keys. + TIMEOUT = 24.hours.to_i + + WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze + local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2] + local existing = tonumber(redis.call("get", key)) + + if existing == nil or value > existing then + redis.call("set", key, value) + redis.call("expire", key, ttl) + return true + else + return false + end + EOF + + # Reads a cache key. + # + # If the key exists and has a non-empty value its TTL is refreshed + # automatically. + # + # raw_key - The cache key to read. + # timeout - The new timeout of the key if the key is to be refreshed. + def self.read(raw_key, timeout: TIMEOUT) + key = cache_key_for(raw_key) + value = Redis::Cache.with { |redis| redis.get(key) } + + if value.present? + # We refresh the expiration time so frequently used keys stick + # around, removing the need for querying the database as much as + # possible. + # + # A key may be empty when we looked up a GitHub user (for example) but + # did not find a matching GitLab user. In that case we _don't_ want to + # refresh the TTL so we automatically pick up the right data when said + # user were to register themselves on the GitLab instance. + Redis::Cache.with { |redis| redis.expire(key, timeout) } + end + + value + end + + # Reads an integer from the cache, or returns nil if no value was found. + # + # See Caching.read for more information. + def self.read_integer(raw_key, timeout: TIMEOUT) + value = read(raw_key, timeout: timeout) + + value.to_i if value.present? + end + + # Sets a cache key to the given value. + # + # key - The cache key to write. + # value - The value to set. + # timeout - The time after which the cache key should expire. + def self.write(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.set(key, value, ex: timeout) + end + + value + end + + # Adds a value to a set. + # + # raw_key - The key of the set to add the value to. + # value - The value to add to the set. + # timeout - The new timeout of the key. + def self.set_add(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.multi do |m| + m.sadd(key, value) + m.expire(key, timeout) + end + end + end + + # Returns true if the given value is present in the set. + # + # raw_key - The key of the set to check. + # value - The value to check for. + def self.set_includes?(raw_key, value) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.sismember(key, value) + end + end + + # Sets multiple keys to a given value. + # + # mapping - A Hash mapping the cache keys to their values. + # timeout - The time after which the cache key should expire. + def self.write_multiple(mapping, timeout: TIMEOUT) + Redis::Cache.with do |redis| + redis.multi do |multi| + mapping.each do |raw_key, value| + multi.set(cache_key_for(raw_key), value, ex: timeout) + end + end + end + end + + # Sets the expiration time of a key. + # + # raw_key - The key for which to change the timeout. + # timeout - The new timeout. + def self.expire(raw_key, timeout) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.expire(key, timeout) + end + end + + # Sets a key to the given integer but only if the existing value is + # smaller than the given value. + # + # This method uses a Lua script to ensure the read and write are atomic. + # + # raw_key - The key to set. + # value - The new value for the key. + # timeout - The key timeout in seconds. + # + # Returns true when the key was overwritten, false otherwise. + def self.write_if_greater(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + val = Redis::Cache.with do |redis| + redis + .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout]) + end + + val ? true : false + end + + def self.cache_key_for(raw_key) + "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 0550f9695bd..844530b1ea7 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -1,147 +1,184 @@ +# frozen_string_literal: true + module Gitlab module GithubImport + # HTTP client for interacting with the GitHub API. + # + # This class is basically a fancy wrapped around Octokit while adding some + # functionality to deal with rate limiting and parallel imports. Usage is + # mostly the same as Octokit, for example: + # + # client = GithubImport::Client.new('hunter2') + # + # client.labels.each do |label| + # puts label.name + # end class Client - GITHUB_SAFE_REMAINING_REQUESTS = 100 - GITHUB_SAFE_SLEEP_TIME = 500 + attr_reader :octokit - attr_reader :access_token, :host, :api_version + # A single page of data and the corresponding page number. + Page = Struct.new(:objects, :number) - def initialize(access_token, host: nil, api_version: 'v3') - @access_token = access_token - @host = host.to_s.sub(%r{/+\z}, '') - @api_version = api_version - @users = {} + # The minimum number of requests we want to keep available. + # + # We don't use a value of 0 as multiple threads may be using the same + # token in parallel. This could result in all of them hitting the GitHub + # rate limit at once. The threshold is put in place to not hit the limit + # in most cases. + RATE_LIMIT_THRESHOLD = 50 - if access_token - ::Octokit.auto_paginate = false - end + # token - The GitHub API token to use. + # + # per_page - The number of objects that should be displayed per page. + # + # parallel - When set to true hitting the rate limit will result in a + # dedicated error being raised. When set to `false` we will + # instead just `sleep()` until the rate limit is reset. Setting + # this value to `true` for parallel importing is crucial as + # otherwise hitting the rate limit will result in a thread + # being blocked in a `sleep()` call for up to an hour. + def initialize(token, per_page: 100, parallel: true) + @octokit = Octokit::Client.new(access_token: token, per_page: per_page) + @parallel = parallel end - def api - @api ||= ::Octokit::Client.new( - access_token: access_token, - api_endpoint: api_endpoint, - # If there is no config, we're connecting to github.com and we - # should verify ssl. - connection_options: { - ssl: { verify: config ? config['verify_ssl'] : true } - } - ) + def parallel? + @parallel end - def client - unless config - raise Projects::ImportService::Error, - 'OAuth configuration for GitHub missing.' - end - - @client ||= ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + # Returns the details of a GitHub user. + # + # username - The username of the user. + def user(username) + with_rate_limit { octokit.user(username) } end - def authorize_url(redirect_uri) - client.auth_code.authorize_url({ - redirect_uri: redirect_uri, - scope: "repo, user, user:email" - }) + # Returns the details of a GitHub repository. + # + # name - The path (in the form `owner/repository`) of the repository. + def repository(name) + with_rate_limit { octokit.repo(name) } end - def get_token(code) - client.auth_code.get_token(code).token + def labels(*args) + each_object(:labels, *args) end - def method_missing(method, *args, &block) - if api.respond_to?(method) - request(method, *args, &block) - else - super(method, *args, &block) - end + def milestones(*args) + each_object(:milestones, *args) end - def respond_to?(method) - api.respond_to?(method) || super + def releases(*args) + each_object(:releases, *args) end - def user(login) - return nil unless login.present? - return @users[login] if @users.key?(login) + # Fetches data from the GitHub API and yields a Page object for every page + # of data, without loading all of them into memory. + # + # method - The Octokit method to use for getting the data. + # args - Arguments to pass to the Octokit method. + # + # rubocop: disable GitlabSecurity/PublicSend + def each_page(method, *args, &block) + return to_enum(__method__, method, *args) unless block_given? - @users[login] = api.user(login) - end + page = + if args.last.is_a?(Hash) && args.last[:page] + args.last[:page] + else + 1 + end - private + collection = with_rate_limit { octokit.public_send(method, *args) } + next_url = octokit.last_response.rels[:next] - def api_endpoint - if host.present? && api_version.present? - "#{host}/api/#{api_version}" - else - github_options[:site] + yield Page.new(collection, page) + + while next_url + response = with_rate_limit { next_url.get } + next_url = response.rels[:next] + + yield Page.new(response.data, page += 1) end end - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } - end + # Iterates over all of the objects for the given method (e.g. `:labels`). + # + # method - The method to send to Octokit for querying data. + # args - Any arguments to pass to the Octokit method. + def each_object(method, *args, &block) + return to_enum(__method__, method, *args) unless block_given? - def github_options - if config - config["args"]["client_options"].deep_symbolize_keys - else - OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + each_page(method, *args) do |page| + page.objects.each do |object| + yield object + end end end - def rate_limit - api.rate_limit! - # GitHub Rate Limit API returns 404 when the rate limit is - # disabled. In this case we just want to return gracefully - # instead of spitting out an error. - rescue Octokit::NotFound - nil - end + # Yields the supplied block, responding to any rate limit errors. + # + # The exact strategy used for handling rate limiting errors depends on + # whether we are running in parallel mode or not. For more information see + # `#rate_or_wait_for_rate_limit`. + def with_rate_limit + request_count_counter.increment - def has_rate_limit? - return @has_rate_limit if defined?(@has_rate_limit) + raise_or_wait_for_rate_limit unless requests_remaining? - @has_rate_limit = rate_limit.present? - end + begin + yield + rescue Octokit::TooManyRequests + raise_or_wait_for_rate_limit - def rate_limit_exceed? - has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + # This retry will only happen when running in sequential mode as we'll + # raise an error in parallel mode. + retry + end end - def rate_limit_sleep_time - rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + # Returns `true` if we're still allowed to perform API calls. + def requests_remaining? + remaining_requests > RATE_LIMIT_THRESHOLD end - def request(method, *args, &block) - sleep rate_limit_sleep_time if rate_limit_exceed? - - data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend - return data unless data.is_a?(Array) + def remaining_requests + octokit.rate_limit.remaining + end - last_response = api.last_response + def raise_or_wait_for_rate_limit + rate_limit_counter.increment - if block_given? - yield data - # api.last_response could change while we're yielding (e.g. fetching labels for each PR) - # so we cache our own last response - each_response_page(last_response, &block) + if parallel? + raise RateLimitError else - each_response_page(last_response) { |page| data.concat(page) } - data + sleep(rate_limit_resets_in) end end - def each_response_page(last_response) - while last_response.rels[:next] - sleep rate_limit_sleep_time if rate_limit_exceed? - last_response = last_response.rels[:next].get - yield last_response.data if last_response.data.is_a?(Array) - end + def rate_limit_resets_in + # We add a few seconds to the rate limit so we don't _immediately_ + # resume when the rate limit resets as this may result in us performing + # a request before GitHub has a chance to reset the limit. + octokit.rate_limit.resets_in + 5 + end + + def respond_to_missing?(method, include_private = false) + octokit.respond_to?(method, include_private) + end + + def rate_limit_counter + @rate_limit_counter ||= Gitlab::Metrics.counter( + :github_importer_rate_limit_hits, + 'The number of times we hit the GitHub rate limit when importing projects' + ) + end + + def request_count_counter + @request_counter ||= Gitlab::Metrics.counter( + :github_importer_request_count, + 'The number of GitHub API calls performed when importing projects' + ) end end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb deleted file mode 100644 index 8911b81ec9a..00000000000 --- a/lib/gitlab/github_import/comment_formatter.rb +++ /dev/null @@ -1,69 +0,0 @@ -module Gitlab - module GithubImport - class CommentFormatter < BaseFormatter - attr_writer :author_id - - def attributes - { - project: project, - note: note, - commit_id: raw_data.commit_id, - line_code: line_code, - author_id: author_id, - type: type, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at - } - end - - private - - def author - @author ||= UserFormatter.new(client, raw_data.user) - end - - def author_id - author.gitlab_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def line_code - return unless on_diff? - - parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) - generate_line_code(parsed_lines.to_a.last) - end - - def generate_line_code(line) - Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) - end - - def on_diff? - diff_hunk.present? - end - - def diff_hunk - raw_data.diff_hunk - end - - def file_path - raw_data.path - end - - def note - if author.gitlab_id - body - else - formatter.author_line(author.login) + body - end - end - - def type - 'LegacyDiffNote' if on_diff? - end - end - end -end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb deleted file mode 100644 index b8c07460ebb..00000000000 --- a/lib/gitlab/github_import/importer.rb +++ /dev/null @@ -1,329 +0,0 @@ -module Gitlab - module GithubImport - class Importer - include Gitlab::ShellAdapter - - attr_reader :errors, :project, :repo, :repo_url - - def initialize(project) - @project = project - @repo = project.import_source - @repo_url = project.import_url - @errors = [] - @labels = {} - end - - def client - return @client if defined?(@client) - unless credentials - raise Projects::ImportService::Error, - "Unable to find project import data credentials for project ID: #{@project.id}" - end - - opts = {} - # Gitea plan to be GitHub compliant - if project.gitea_import? - uri = URI.parse(project.import_url) - host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') - opts = { - host: host, - api_version: 'v1' - } - end - - @client = Client.new(credentials[:user], opts) - end - - def execute - # The ordering of importing is important here due to the way GitHub structures their data - # 1. Labels are required by other items while not having a dependency on anything else - # so need to be first - # 2. Pull requests must come before issues. Every pull request is also an issue but not - # all issues are pull requests. Only the issue entity has labels defined in GitHub. GitLab - # doesn't structure data like this so we need to make sure that we've created the MRs - # before we attempt to add the labels defined in the GitHub issue for the related, already - # imported, pull request - import_labels - import_milestones - import_pull_requests - import_issues - import_comments(:issues) - import_comments(:pull_requests) - import_wiki - - # Gitea doesn't have a Release API yet - # See https://github.com/go-gitea/gitea/issues/330 - unless project.gitea_import? - import_releases - end - - handle_errors - - true - end - - private - - def credentials - return @credentials if defined?(@credentials) - - @credentials = project.import_data ? project.import_data.credentials : nil - end - - def handle_errors - return unless errors.any? - - project.update_column(:import_error, { - message: 'The remote data could not be fully imported.', - errors: errors - }.to_json) - end - - def import_labels - fetch_resources(:labels, repo, per_page: 100) do |labels| - labels.each do |raw| - begin - gh_label = LabelFormatter.new(project, raw) - gh_label.create! - rescue => e - errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } - end - end - end - - cache_labels! - end - - def import_milestones - fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| - milestones.each do |raw| - begin - gh_milestone = MilestoneFormatter.new(project, raw) - gh_milestone.create! - rescue => e - errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } - end - end - end - end - - def import_issues - fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| - issues.each do |raw| - gh_issue = IssueFormatter.new(project, raw, client) - - begin - issuable = - if gh_issue.pull_request? - MergeRequest.find_by(target_project_id: project.id, iid: gh_issue.number) - else - gh_issue.create! - end - - apply_labels(issuable, raw) - rescue => e - errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } - end - end - end - end - - def import_pull_requests - fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| - pull_requests.each do |raw| - gh_pull_request = PullRequestFormatter.new(project, raw, client) - - next unless gh_pull_request.valid? - - begin - restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists? - restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists? - - merge_request = gh_pull_request.create! - - # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage - if project.gitea_import? - apply_labels(merge_request, raw) - end - rescue => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } - ensure - clean_up_restored_branches(gh_pull_request) - end - end - end - - project.repository.after_remove_branch - end - - def restore_source_branch(pull_request) - project.repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) - end - - def restore_target_branch(pull_request) - project.repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) - end - - def remove_branch(name) - project.repository.delete_branch(name) - rescue Gitlab::Git::Repository::DeleteBranchFailed - errors << { type: :remove_branch, name: name } - end - - def clean_up_restored_branches(pull_request) - return if pull_request.opened? - - remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? - remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? - end - - def apply_labels(issuable, raw) - return unless raw.labels.count > 0 - - label_ids = raw.labels - .map { |attrs| @labels[attrs.name] } - .compact - - issuable.update_attribute(:label_ids, label_ids) - end - - def import_comments(issuable_type) - resource_type = "#{issuable_type}_comments".to_sym - - # Two notes here: - # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note, - # compare it against every comment in the current imported page until we find match, and that's where start importing - # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns - # only comments on diffs, so select last note not based on noteable_type but on line_code - line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL' - last_note = project.notes.where("line_code IS #{line_code_is}").last - - fetch_resources(resource_type, repo, per_page: 100) do |comments| - if last_note - discard_inserted_comments(comments, last_note) - last_note = nil - end - - create_comments(comments) - end - end - - def create_comments(comments) - ActiveRecord::Base.no_touching do - comments.each do |raw| - begin - comment = CommentFormatter.new(project, raw, client) - - # GH does not return info about comment's parent, so we guess it by checking its URL! - *_, parent, iid = URI(raw.html_url).path.split('/') - - issuable = if parent == 'issues' - Issue.find_by(project_id: project.id, iid: iid) - else - MergeRequest.find_by(target_project_id: project.id, iid: iid) - end - - next unless issuable - - issuable.notes.create!(comment.attributes) - rescue => e - errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } - end - end - end - end - - def discard_inserted_comments(comments, last_note) - last_note_attrs = nil - - cut_off_index = comments.find_index do |raw| - comment = CommentFormatter.new(project, raw) - comment_attrs = comment.attributes - last_note_attrs ||= last_note.slice(*comment_attrs.keys) - - comment_attrs.with_indifferent_access == last_note_attrs - end - - # No matching resource in the collection, which means we got halted right on the end of the last page, so all good - return unless cut_off_index - - # Otherwise, remove the resources we've already inserted - comments.shift(cut_off_index + 1) - end - - def import_wiki - unless project.wiki.repository_exists? - wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) - end - rescue Gitlab::Shell::Error => e - # GitHub error message when the wiki repo has not been created, - # this means that repo has wiki enabled, but have no pages. So, - # we can skip the import. - if e.message !~ /repository not exported/ - errors << { type: :wiki, errors: e.message } - end - end - - def import_releases - fetch_resources(:releases, repo, per_page: 100) do |releases| - releases.each do |raw| - begin - gh_release = ReleaseFormatter.new(project, raw) - gh_release.create! if gh_release.valid? - rescue => e - errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } - end - end - end - end - - def cache_labels! - project.labels.select(:id, :title).find_each do |label| - @labels[label.title] = label.id - end - end - - def fetch_resources(resource_type, *opts) - return if imported?(resource_type) - - opts.last[:page] = current_page(resource_type) - - client.public_send(resource_type, *opts) do |resources| # rubocop:disable GitlabSecurity/PublicSend - yield resources - increment_page(resource_type) - end - - imported!(resource_type) - end - - def imported?(resource_type) - Rails.cache.read("#{cache_key_prefix}:#{resource_type}:imported") - end - - def imported!(resource_type) - Rails.cache.write("#{cache_key_prefix}:#{resource_type}:imported", true, ex: 1.day) - end - - def increment_page(resource_type) - key = "#{cache_key_prefix}:#{resource_type}:current-page" - - # Rails.cache.increment calls INCRBY directly on the value stored under the key, which is - # a serialized ActiveSupport::Cache::Entry, so it will return an error by Redis, hence this ugly work-around - page = Rails.cache.read(key) - page += 1 - Rails.cache.write(key, page) - - page - end - - def current_page(resource_type) - Rails.cache.fetch("#{cache_key_prefix}:#{resource_type}:current-page", ex: 1.day) { 1 } - end - - def cache_key_prefix - @cache_key_prefix ||= "github-import:#{project.id}" - end - end - end -end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb new file mode 100644 index 00000000000..8274f37d358 --- /dev/null +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class DiffNoteImporter + attr_reader :note, :project, :client, :user_finder + + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note, project, client) + @note = note + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + return unless (mr_id = find_merge_request_id) + + author_id, author_found = user_finder.author_id_for(note) + + note_body = + MarkdownText.format(note.note, note.author, author_found) + + attributes = { + noteable_type: 'MergeRequest', + noteable_id: mr_id, + project_id: project.id, + author_id: author_id, + note: note_body, + system: false, + commit_id: note.commit_id, + line_code: note.line_code, + type: 'LegacyDiffNote', + created_at: note.created_at, + updated_at: note.updated_at, + st_diff: note.diff_hash.to_yaml + } + + # It's possible that during an import we'll insert tens of thousands + # of diff notes. If we were to use the Note/LegacyDiffNote model here + # we'd also have to run additional queries for both validations and + # callbacks, putting a lot of pressure on the database. + # + # To work around this we're using bulk_insert with a single row. This + # allows us to efficiently insert data (even if it's just 1 row) + # without having to use all sorts of hacks to disable callbacks. + Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note. + end + + # Returns the ID of the merge request this note belongs to. + def find_merge_request_id + GithubImport::IssuableFinder.new(project, note).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb new file mode 100644 index 00000000000..966f12c5c2f --- /dev/null +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class DiffNotesImporter + include ParallelScheduling + + def representation_class + Representation::DiffNote + end + + def importer_class + DiffNoteImporter + end + + def sidekiq_worker_class + ImportDiffNoteWorker + end + + def collection_method + :pull_requests_comments + end + + def id_for_already_imported_cache(note) + note.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb b/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb new file mode 100644 index 00000000000..bad064b76c8 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueAndLabelLinksImporter + attr_reader :issue, :project, :client + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + end + + def execute + IssueImporter.import_if_issue(issue, project, client) + LabelLinksImporter.new(issue, project, client).execute + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb new file mode 100644 index 00000000000..31fefebf787 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueImporter + attr_reader :project, :issue, :client, :user_finder, :milestone_finder, + :issuable_finder + + # Imports an issue if it's a regular issue and not a pull request. + def self.import_if_issue(issue, project, client) + new(issue, project, client).execute unless issue.pull_request? + end + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + @milestone_finder = MilestoneFinder.new(project) + @issuable_finder = GithubImport::IssuableFinder.new(project, issue) + end + + def execute + Issue.transaction do + if (issue_id = create_issue) + create_assignees(issue_id) + issuable_finder.cache_database_id(issue_id) + end + end + end + + # Creates a new GitLab issue for the current GitHub issue. + # + # Returns the ID of the created issue as an Integer. If the issue + # couldn't be created this method will return `nil` instead. + def create_issue + author_id, author_found = user_finder.author_id_for(issue) + + description = + MarkdownText.format(issue.description, issue.author, author_found) + + attributes = { + iid: issue.iid, + title: issue.truncated_title, + author_id: author_id, + project_id: project.id, + description: description, + milestone_id: milestone_finder.id_for(issue), + state: issue.state, + created_at: issue.created_at, + updated_at: issue.updated_at + } + + GithubImport.insert_and_return_id(attributes, project.issues) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the issue. + end + + # Stores all issue assignees in the database. + # + # issue_id - The ID of the created issue. + def create_assignees(issue_id) + assignees = [] + + issue.assignees.each do |assignee| + if (user_id = user_finder.user_id_for(assignee)) + assignees << { issue_id: issue_id, user_id: user_id } + end + end + + Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb new file mode 100644 index 00000000000..ac6d0666b3a --- /dev/null +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssuesImporter + include ParallelScheduling + + def importer_class + IssueAndLabelLinksImporter + end + + def representation_class + Representation::Issue + end + + def sidekiq_worker_class + ImportIssueWorker + end + + def collection_method + :issues + end + + def id_for_already_imported_cache(issue) + issue.number + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb new file mode 100644 index 00000000000..2001b7e3482 --- /dev/null +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LabelLinksImporter + attr_reader :issue, :project, :client, :label_finder + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + @label_finder = LabelFinder.new(project) + end + + def execute + create_labels + end + + def create_labels + time = Time.zone.now + rows = [] + target_id = find_target_id + + issue.label_names.each do |label_name| + # Although unlikely it's technically possible for an issue to be + # given a label that was created and assigned after we imported all + # the project's labels. + next unless (label_id = label_finder.id_for(label_name)) + + rows << { + label_id: label_id, + target_id: target_id, + target_type: issue.issuable_type, + created_at: time, + updated_at: time + } + end + + Gitlab::Database.bulk_insert(LabelLink.table_name, rows) + end + + def find_target_id + GithubImport::IssuableFinder.new(project, issue).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb new file mode 100644 index 00000000000..a73033d35ba --- /dev/null +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LabelsImporter + include BulkImporting + + attr_reader :project, :client, :existing_labels + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(project, client) + @project = project + @client = client + @existing_labels = project.labels.pluck(:title).to_set + end + + def execute + bulk_insert(Label, build_labels) + build_labels_cache + end + + def build_labels + build_database_rows(each_label) + end + + def already_imported?(label) + existing_labels.include?(label.name) + end + + def build_labels_cache + LabelFinder.new(project).build_cache + end + + def build(label) + time = Time.zone.now + + { + title: label.name, + color: '#' + label.color, + project_id: project.id, + type: 'ProjectLabel', + created_at: time, + updated_at: time + } + end + + def each_label + client.labels(project.import_source) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb new file mode 100644 index 00000000000..c53480e828a --- /dev/null +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class MilestonesImporter + include BulkImporting + + attr_reader :project, :client, :existing_milestones + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + @existing_milestones = project.milestones.pluck(:iid).to_set + end + + def execute + bulk_insert(Milestone, build_milestones) + build_milestones_cache + end + + def build_milestones + build_database_rows(each_milestone) + end + + def already_imported?(milestone) + existing_milestones.include?(milestone.number) + end + + def build_milestones_cache + MilestoneFinder.new(project).build_cache + end + + def build(milestone) + { + iid: milestone.number, + title: milestone.title, + description: milestone.description, + project_id: project.id, + state: state_for(milestone), + created_at: milestone.created_at, + updated_at: milestone.updated_at + } + end + + def state_for(milestone) + milestone.state == 'open' ? :active : :closed + end + + def each_milestone + client.milestones(project.import_source, state: 'all') + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb new file mode 100644 index 00000000000..c890f2df360 --- /dev/null +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NoteImporter + attr_reader :note, :project, :client, :user_finder + + # note - An instance of `Gitlab::GithubImport::Representation::Note`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note, project, client) + @note = note + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + return unless (noteable_id = find_noteable_id) + + author_id, author_found = user_finder.author_id_for(note) + + note_body = + MarkdownText.format(note.note, note.author, author_found) + + attributes = { + noteable_type: note.noteable_type, + noteable_id: noteable_id, + project_id: project.id, + author_id: author_id, + note: note_body, + system: false, + created_at: note.created_at, + updated_at: note.updated_at + } + + # We're using bulk_insert here so we can bypass any validations and + # callbacks. Running these would result in a lot of unnecessary SQL + # queries being executed when importing large projects. + Gitlab::Database.bulk_insert(Note.table_name, [attributes]) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note. + end + + # Returns the ID of the issue or merge request to create the note for. + def find_noteable_id + GithubImport::IssuableFinder.new(project, note).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb new file mode 100644 index 00000000000..5aec760ea5f --- /dev/null +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NotesImporter + include ParallelScheduling + + def importer_class + NoteImporter + end + + def representation_class + Representation::Note + end + + def sidekiq_worker_class + ImportNoteWorker + end + + def collection_method + :issues_comments + end + + def id_for_already_imported_cache(note) + note.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb new file mode 100644 index 00000000000..49d859f9624 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestImporter + attr_reader :pull_request, :project, :client, :user_finder, + :milestone_finder, :issuable_finder + + # pull_request - An instance of + # `Gitlab::GithubImport::Representation::PullRequest`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(pull_request, project, client) + @pull_request = pull_request + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + @milestone_finder = MilestoneFinder.new(project) + @issuable_finder = + GithubImport::IssuableFinder.new(project, pull_request) + end + + def execute + if (mr_id = create_merge_request) + issuable_finder.cache_database_id(mr_id) + end + end + + # Creates the merge request and returns its ID. + # + # This method will return `nil` if the merge request could not be + # created. + def create_merge_request + author_id, author_found = user_finder.author_id_for(pull_request) + + description = MarkdownText + .format(pull_request.description, pull_request.author, author_found) + + # This work must be wrapped in a transaction as otherwise we can leave + # behind incomplete data in the event of an error. This can then lead + # to duplicate key errors when jobs are retried. + MergeRequest.transaction do + attributes = { + iid: pull_request.iid, + title: pull_request.truncated_title, + description: description, + source_project_id: project.id, + target_project_id: project.id, + source_branch: pull_request.formatted_source_branch, + target_branch: pull_request.target_branch, + state: pull_request.state, + milestone_id: milestone_finder.id_for(pull_request), + author_id: author_id, + assignee_id: user_finder.assignee_id_for(pull_request), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + # When creating merge requests there are a lot of hooks that may + # run, for many different reasons. Many of these hooks (e.g. the + # ones used for rendering Markdown) are completely unnecessary and + # may even lead to transaction timeouts. + # + # To ensure importing pull requests has a minimal impact and can + # complete in a reasonable time we bypass all the hooks by inserting + # the row and then retrieving it. We then only perform the + # additional work that is strictly necessary. + merge_request_id = GithubImport + .insert_and_return_id(attributes, project.merge_requests) + + merge_request = project.merge_requests.find(merge_request_id) + + # These fields are set so we can create the correct merge request + # diffs. + merge_request.source_branch_sha = pull_request.source_branch_sha + merge_request.target_branch_sha = pull_request.target_branch_sha + + merge_request.keep_around_commit + merge_request.merge_request_diffs.create + + merge_request.id + end + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the merge request. + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb new file mode 100644 index 00000000000..5437e32e9f1 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestsImporter + include ParallelScheduling + + def importer_class + PullRequestImporter + end + + def representation_class + Representation::PullRequest + end + + def sidekiq_worker_class + ImportPullRequestWorker + end + + def id_for_already_imported_cache(pr) + pr.number + end + + def each_object_to_import + super do |pr| + update_repository if update_repository?(pr) + yield pr + end + end + + def update_repository + # We set this column _before_ fetching the repository, and this is + # deliberate. If we were to update this column after the fetch we may + # miss out on changes pushed during the fetch or between the fetch and + # updating the timestamp. + project.update_column(:last_repository_updated_at, Time.zone.now) + + project.repository.fetch_remote('github', forced: false) + + pname = project.path_with_namespace + + Rails.logger + .info("GitHub importer finished updating repository for #{pname}") + + repository_updates_counter.increment(project: pname) + end + + def update_repository?(pr) + last_update = project.last_repository_updated_at || project.created_at + + return false if pr.updated_at < last_update + + # PRs may be updated without there actually being new commits, thus we + # check to make sure we only re-fetch if truly necessary. + !(commit_exists?(pr.head.sha) && commit_exists?(pr.base.sha)) + end + + def commit_exists?(sha) + project.repository.lookup(sha) + true + rescue Rugged::Error + false + end + + def collection_method + :pull_requests + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + + def repository_updates_counter + @repository_updates_counter ||= Gitlab::Metrics.counter( + :github_importer_repository_updates, + 'The number of times repositories have to be updated again' + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb new file mode 100644 index 00000000000..100f459fdcc --- /dev/null +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleasesImporter + include BulkImporting + + attr_reader :project, :client, :existing_tags + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + @existing_tags = project.releases.pluck(:tag).to_set + end + + def execute + bulk_insert(Release, build_releases) + end + + def build_releases + build_database_rows(each_release) + end + + def already_imported?(release) + existing_tags.include?(release.tag_name) + end + + def build(release) + { + tag: release.tag_name, + description: description_for(release), + created_at: release.created_at, + updated_at: release.updated_at, + project_id: project.id + } + end + + def each_release + client.releases(project.import_source) + end + + def description_for(release) + if release.body.present? + release.body + else + "Release for tag #{release.tag_name}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb new file mode 100644 index 00000000000..0b67fc8db73 --- /dev/null +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class RepositoryImporter + include Gitlab::ShellAdapter + + attr_reader :project, :client + + def initialize(project, client) + @project = project + @client = client + end + + # Returns true if we should import the wiki for the project. + def import_wiki? + client.repository(project.import_source)&.has_wiki && + !project.wiki_repository_exists? + end + + # Imports the repository data. + # + # This method will return true if the data was imported successfully or + # the repository had already been imported before. + def execute + imported = + # It's possible a repository has already been imported when running + # this code, e.g. because we had to retry this job after + # `import_wiki?` raised a rate limit error. In this case we'll skip + # re-importing the main repository. + if project.repository.empty_repo? + import_repository + else + true + end + + update_clone_time if imported + + imported = import_wiki_repository if import_wiki? && imported + + imported + end + + def import_repository + project.ensure_repository + + configure_repository_remote + + project.repository.fetch_remote('github', forced: true) + + true + rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e + fail_import("Failed to import the repository: #{e.message}") + end + + def configure_repository_remote + return if project.repository.remote_exists?('github') + + project.repository.add_remote('github', project.import_url) + project.repository.set_import_remote_as_mirror('github') + + project.repository.add_remote_fetch_config( + 'github', + '+refs/pull/*/head:refs/merge-requests/*/head' + ) + end + + def import_wiki_repository + wiki_path = "#{project.disk_path}.wiki" + wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') + storage_path = project.repository_storage_path + + gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) + + true + rescue Gitlab::Shell::Error => e + if e.message !~ /repository not exported/ + fail_import("Failed to import the wiki: #{e.message}") + else + true + end + end + + def update_clone_time + project.update_column(:last_repository_updated_at, Time.zone.now) + end + + def fail_import(message) + project.mark_import_as_failed(message) + false + end + end + end + end +end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb new file mode 100644 index 00000000000..211915f1d87 --- /dev/null +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # IssuableFinder can be used for caching and retrieving database IDs for + # issuable objects such as issues and pull requests. By caching these IDs we + # remove the need for running a lot of database queries when importing + # GitHub projects. + class IssuableFinder + attr_reader :project, :object + + # The base cache key to use for storing/retrieving issuable IDs. + CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}'.freeze + + # project - An instance of `Project`. + # object - The object to look up or set a database ID for. + def initialize(project, object) + @project = project + @object = object + end + + # Returns the database ID for the object. + # + # This method will return `nil` if no ID could be found. + def database_id + val = Caching.read(cache_key) + + val.to_i if val.present? + end + + # Associates the given database ID with the current object. + # + # database_id - The ID of the corresponding database row. + def cache_database_id(database_id) + Caching.write(cache_key, database_id) + end + + private + + def cache_key + CACHE_KEY % { + project: project.id, + type: cache_key_type, + iid: cache_key_iid + } + end + + # Returns the identifier to use for cache keys. + # + # For issues and pull requests this will be "Issue" or "MergeRequest" + # respectively. For diff notes this will return "MergeRequest", for + # regular notes it will either return "Issue" or "MergeRequest" depending + # on what type of object the note belongs to. + def cache_key_type + if object.respond_to?(:issuable_type) + object.issuable_type + elsif object.respond_to?(:noteable_type) + object.noteable_type + else + raise( + TypeError, + "Instances of #{object.class} are not supported" + ) + end + end + + def cache_key_iid + if object.respond_to?(:noteable_id) + object.noteable_id + elsif object.respond_to?(:iid) + object.iid + else + raise( + TypeError, + "Instances of #{object.class} are not supported" + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb deleted file mode 100644 index 27b171d6ddb..00000000000 --- a/lib/gitlab/github_import/issuable_formatter.rb +++ /dev/null @@ -1,66 +0,0 @@ -module Gitlab - module GithubImport - class IssuableFormatter < BaseFormatter - attr_writer :assignee_id, :author_id - - def project_association - raise NotImplementedError - end - - delegate :number, to: :raw_data - - def find_condition - { iid: number } - end - - private - - def state - raw_data.state == 'closed' ? 'closed' : 'opened' - end - - def assigned? - raw_data.assignee.present? - end - - def author - @author ||= UserFormatter.new(client, raw_data.user) - end - - def author_id - @author_id ||= author.gitlab_id || project.creator_id - end - - def assignee - if assigned? - @assignee ||= UserFormatter.new(client, raw_data.assignee) - end - end - - def assignee_id - return @assignee_id if defined?(@assignee_id) - - @assignee_id = assignee.try(:gitlab_id) - end - - def body - raw_data.body || "" - end - - def description - if author.gitlab_id - body - else - formatter.author_line(author.login) + body - end - end - - def milestone - if raw_data.milestone.present? - milestone = MilestoneFormatter.new(project, raw_data.milestone) - project.milestones.find_by(milestone.find_condition) - end - end - end - end -end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb deleted file mode 100644 index 977cd0423ba..00000000000 --- a/lib/gitlab/github_import/issue_formatter.rb +++ /dev/null @@ -1,32 +0,0 @@ -module Gitlab - module GithubImport - class IssueFormatter < IssuableFormatter - def attributes - { - iid: number, - project: project, - milestone: milestone, - title: raw_data.title, - description: description, - state: state, - author_id: author_id, - assignee_ids: Array(assignee_id), - created_at: raw_data.created_at, - updated_at: raw_data.updated_at - } - end - - def has_comments? - raw_data.comments > 0 - end - - def project_association - :issues - end - - def pull_request? - raw_data.pull_request.present? - end - end - end -end diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb new file mode 100644 index 00000000000..9be071141db --- /dev/null +++ b/lib/gitlab/github_import/label_finder.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class LabelFinder + attr_reader :project + + # The base cache key to use for storing/retrieving label IDs. + CACHE_KEY = 'github-import/label-finder/%{project}/%{name}'.freeze + + # project - An instance of `Project`. + def initialize(project) + @project = project + end + + # Returns the label ID for the given name. + def id_for(name) + Caching.read_integer(cache_key_for(name)) + end + + def build_cache + mapping = @project + .labels + .pluck(:id, :name) + .each_with_object({}) do |(id, name), hash| + hash[cache_key_for(name)] = id + end + + Caching.write_multiple(mapping) + end + + def cache_key_for(name) + CACHE_KEY % { project: project.id, name: name } + end + end + end +end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb deleted file mode 100644 index 211ccdc51bb..00000000000 --- a/lib/gitlab/github_import/label_formatter.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module GithubImport - class LabelFormatter < BaseFormatter - def attributes - { - project: project, - title: title, - color: color - } - end - - def project_association - :labels - end - - def create! - params = attributes.except(:project) - service = ::Labels::FindOrCreateService.new(nil, project, params) - label = service.execute(skip_authorization: true) - - raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? - - label - end - - private - - def color - "##{raw_data.color}" - end - - def title - raw_data.name - end - end - end -end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb new file mode 100644 index 00000000000..b25c4f7becf --- /dev/null +++ b/lib/gitlab/github_import/markdown_text.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class MarkdownText + attr_reader :text, :author, :exists + + def self.format(*args) + new(*args).to_s + end + + # text - The Markdown text as a String. + # author - An instance of `Gitlab::GithubImport::Representation::User` + # exists - Boolean that indicates the user exists in the GitLab database. + def initialize(text, author, exists = false) + @text = text + @author = author + @exists = exists + end + + def to_s + if exists + text + else + "*Created by: #{author.login}*\n\n#{text}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb new file mode 100644 index 00000000000..208d15dc144 --- /dev/null +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class MilestoneFinder + attr_reader :project + + # The base cache key to use for storing/retrieving milestone IDs. + CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}'.freeze + + # project - An instance of `Project` + def initialize(project) + @project = project + end + + # issuable - An instance of `Gitlab::GithubImport::Representation::Issue` + # or `Gitlab::GithubImport::Representation::PullRequest`. + def id_for(issuable) + return unless issuable.milestone_number + + Caching.read_integer(cache_key_for(issuable.milestone_number)) + end + + def build_cache + mapping = @project + .milestones + .pluck(:id, :iid) + .each_with_object({}) do |(id, iid), hash| + hash[cache_key_for(iid)] = id + end + + Caching.write_multiple(mapping) + end + + def cache_key_for(iid) + CACHE_KEY % { project: project.id, iid: iid } + end + end + end +end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb deleted file mode 100644 index dd782eff059..00000000000 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ /dev/null @@ -1,40 +0,0 @@ -module Gitlab - module GithubImport - class MilestoneFormatter < BaseFormatter - def attributes - { - iid: number, - project: project, - title: raw_data.title, - description: raw_data.description, - due_date: raw_data.due_on, - state: state, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at - } - end - - def project_association - :milestones - end - - def find_condition - { iid: number } - end - - def number - if project.gitea_import? - raw_data.id - else - raw_data.number - end - end - - private - - def state - raw_data.state == 'closed' ? 'closed' : 'active' - end - end - end -end diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb new file mode 100644 index 00000000000..c3db2d0b469 --- /dev/null +++ b/lib/gitlab/github_import/page_counter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # PageCounter can be used to keep track of the last imported page of a + # collection, allowing workers to resume where they left off in the event of + # an error. + class PageCounter + attr_reader :cache_key + + # The base cache key to use for storing the last page number. + CACHE_KEY = 'github-importer/page-counter/%{project}/%{collection}'.freeze + + def initialize(project, collection) + @cache_key = CACHE_KEY % { project: project.id, collection: collection } + end + + # Sets the page number to the given value. + # + # Returns true if the page number was overwritten, false otherwise. + def set(page) + Caching.write_if_greater(cache_key, page) + end + + # Returns the current value from the cache. + def current + Caching.read_integer(cache_key) || 1 + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb new file mode 100644 index 00000000000..81739834b41 --- /dev/null +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # The ParallelImporter schedules the importing of a GitHub project using + # Sidekiq. + class ParallelImporter + attr_reader :project + + def self.async? + true + end + + def initialize(project) + @project = project + end + + def execute + jid = generate_jid + + # The original import JID is the JID of the RepositoryImportWorker job, + # which will be removed once that job completes. Reusing that JID could + # result in StuckImportJobsWorker marking the job as stuck before we get + # to running Stage::ImportRepositoryWorker. + # + # We work around this by setting the JID to a custom generated one, then + # refreshing it in the various stages whenever necessary. + Gitlab::SidekiqStatus + .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + + project.update_column(:import_jid, jid) + + Stage::ImportRepositoryWorker + .perform_async(project.id) + + true + end + + def generate_jid + "github-importer/#{project.id}" + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb new file mode 100644 index 00000000000..d4d1357f5a3 --- /dev/null +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module ParallelScheduling + attr_reader :project, :client, :page_counter, :already_imported_cache_key + + # The base cache key to use for tracking already imported objects. + ALREADY_IMPORTED_CACHE_KEY = + 'github-importer/already-imported/%{project}/%{collection}'.freeze + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + # parallel - When set to true the objects will be imported in parallel. + def initialize(project, client, parallel: true) + @project = project + @client = client + @parallel = parallel + @page_counter = PageCounter.new(project, collection_method) + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def parallel? + @parallel + end + + def execute + retval = + if parallel? + parallel_import + else + sequential_import + end + + # Once we have completed all work we can remove our "already exists" + # cache so we don't put too much pressure on Redis. + # + # We don't immediately remove it since it's technically possible for + # other instances of this job to still run, instead we set the + # expiration time to a lower value. This prevents the other jobs from + # still scheduling duplicates while. Since all work has already been + # completed those jobs will just cycle through any remaining pages while + # not scheduling anything. + Caching.expire(already_imported_cache_key, 15.minutes.to_i) + + retval + end + + # Imports all the objects in sequence in the current thread. + def sequential_import + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + importer_class.new(repr, project, client).execute + end + end + + # Imports all objects in parallel by scheduling a Sidekiq job for every + # individual object. + def parallel_import + waiter = JobWaiter.new + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + sidekiq_worker_class + .perform_async(project.id, repr.to_hash, waiter.key) + + waiter.jobs_remaining += 1 + end + + waiter + end + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + repo = project.import_source + + # We inject the page number here to make sure that all importers always + # start where they left off. Simply starting over wouldn't work for + # repositories with a lot of data (e.g. tens of thousands of comments). + options = collection_options.merge(page: page_counter.current) + + client.each_page(collection_method, repo, options) do |page| + # Technically it's possible that the same work is performed multiple + # times, as Sidekiq doesn't guarantee there will ever only be one + # instance of a job. In such a scenario it's possible for one job to + # have a lower page number (e.g. 5) compared to another (e.g. 10). In + # this case we skip over all the objects until we have caught up, + # reducing the number of duplicate jobs scheduled by the provided + # block. + next unless page_counter.set(page.number) + + page.objects.each do |object| + next if already_imported?(object) + + yield object + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(object) + end + end + end + + # Returns true if the given object has already been imported, false + # otherwise. + # + # object - The object to check. + def already_imported?(object) + id = id_for_already_imported_cache(object) + + Caching.set_includes?(already_imported_cache_key, id) + end + + # Marks the given object as "already imported". + def mark_as_imported(object) + id = id_for_already_imported_cache(object) + + Caching.set_add(already_imported_cache_key, id) + end + + # Returns the ID to use for the cache used for checking if an object has + # already been imported or not. + # + # object - The object we may want to import. + def id_for_already_imported_cache(object) + raise NotImplementedError + end + + # The class used for converting API responses to Hashes when performing + # the import. + def representation_class + raise NotImplementedError + end + + # The class to use for importing objects when importing them sequentially. + def importer_class + raise NotImplementedError + end + + # The Sidekiq worker class used for scheduling the importing of objects in + # parallel. + def sidekiq_worker_class + raise NotImplementedError + end + + # The name of the method to call to retrieve the data to import. + def collection_method + raise NotImplementedError + end + + # Any options to be passed to the method used for retrieving the data to + # import. + def collection_options + {} + end + end + end +end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb deleted file mode 100644 index a55adc9b1c8..00000000000 --- a/lib/gitlab/github_import/project_creator.rb +++ /dev/null @@ -1,52 +0,0 @@ -module Gitlab - module GithubImport - class ProjectCreator - include Gitlab::CurrentSettings - - attr_reader :repo, :name, :namespace, :current_user, :session_data, :type - - def initialize(repo, name, namespace, current_user, session_data, type: 'github') - @repo = repo - @name = name - @namespace = namespace - @current_user = current_user - @session_data = session_data - @type = type - end - - def execute - ::Projects::CreateService.new( - current_user, - name: name, - path: name, - description: repo.description, - namespace_id: namespace.id, - visibility_level: visibility_level, - import_type: type, - import_source: repo.full_name, - import_url: import_url, - skip_wiki: skip_wiki - ).execute - end - - private - - def import_url - repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") - end - - def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility - end - - # - # If the GitHub project repository has wiki, we should not create the - # default wiki. Otherwise the GitHub importer will fail because the wiki - # repository already exist. - # - def skip_wiki - repo.has_wiki? - end - end - end -end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb deleted file mode 100644 index 150afa31432..00000000000 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ /dev/null @@ -1,90 +0,0 @@ -module Gitlab - module GithubImport - class PullRequestFormatter < IssuableFormatter - delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true - delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true - - def attributes - { - iid: number, - title: raw_data.title, - description: description, - source_project: source_branch_project, - source_branch: source_branch_name, - source_branch_sha: source_branch_sha, - target_project: target_branch_project, - target_branch: target_branch_name, - target_branch_sha: target_branch_sha, - state: state, - milestone: milestone, - author_id: author_id, - assignee_id: assignee_id, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at, - imported: true - } - end - - def project_association - :merge_requests - end - - def valid? - source_branch.valid? && target_branch.valid? - end - - def source_branch - @source_branch ||= BranchFormatter.new(project, raw_data.head) - end - - def source_branch_name - @source_branch_name ||= - if cross_project? || !source_branch_exists? - source_branch_name_prefixed - else - source_branch_ref - end - end - - def source_branch_name_prefixed - "gh-#{target_branch_short_sha}/#{number}/#{source_branch_user}/#{source_branch_ref}" - end - - def source_branch_exists? - !cross_project? && source_branch.exists? - end - - def target_branch - @target_branch ||= BranchFormatter.new(project, raw_data.base) - end - - def target_branch_name - @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed - end - - def target_branch_name_prefixed - "gl-#{target_branch_short_sha}/#{number}/#{target_branch_user}/#{target_branch_ref}" - end - - def cross_project? - return true if source_branch_repo.nil? - - source_branch_repo.id != target_branch_repo.id - end - - def opened? - state == 'opened' - end - - private - - def state - if raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - else - super - end - end - end - end -end diff --git a/lib/gitlab/github_import/rate_limit_error.rb b/lib/gitlab/github_import/rate_limit_error.rb new file mode 100644 index 00000000000..cc2de909c29 --- /dev/null +++ b/lib/gitlab/github_import/rate_limit_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Error that will be raised when we're about to reach (or have reached) the + # GitHub API's rate limit. + RateLimitError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb deleted file mode 100644 index 1ad702a6058..00000000000 --- a/lib/gitlab/github_import/release_formatter.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module GithubImport - class ReleaseFormatter < BaseFormatter - def attributes - { - project: project, - tag: raw_data.tag_name, - description: raw_data.body, - created_at: raw_data.created_at, - updated_at: raw_data.created_at - } - end - - def project_association - :releases - end - - def find_condition - { tag: raw_data.tag_name } - end - - def valid? - !raw_data.draft - end - end - end -end diff --git a/lib/gitlab/github_import/representation.rb b/lib/gitlab/github_import/representation.rb new file mode 100644 index 00000000000..639477ef2a2 --- /dev/null +++ b/lib/gitlab/github_import/representation.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + TIMESTAMP_KEYS = %i[created_at updated_at merged_at].freeze + + # Converts a Hash with String based keys to one that can be used by the + # various Representation classes. + # + # Example: + # + # Representation.symbolize_hash('number' => 10) # => { number: 10 } + def self.symbolize_hash(raw_hash = nil) + hash = raw_hash.deep_symbolize_keys + + TIMESTAMP_KEYS.each do |key| + hash[key] = Time.parse(hash[key]) if hash[key].is_a?(String) + end + + hash + end + end + end +end diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb new file mode 100644 index 00000000000..bb7439a0641 --- /dev/null +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class DiffNote + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :noteable_type, :noteable_id, :commit_id, :file_path, + :diff_hunk, :author, :note, :created_at, :updated_at, + :github_id + + NOTEABLE_ID_REGEX = /\/pull\/(?\d+)/i + + # Builds a diff note from a GitHub API response. + # + # note - An instance of `Sawyer::Resource` containing the note details. + def self.from_api_response(note) + matches = note.html_url.match(NOTEABLE_ID_REGEX) + + unless matches + raise( + ArgumentError, + "The note URL #{note.html_url.inspect} is not supported" + ) + end + + user = Representation::User.from_api_response(note.user) if note.user + hash = { + noteable_type: 'MergeRequest', + noteable_id: matches[:iid].to_i, + file_path: note.path, + commit_id: note.commit_id, + diff_hunk: note.diff_hunk, + author: user, + note: note.body, + created_at: note.created_at, + updated_at: note.updated_at, + github_id: note.id + } + + new(hash) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + def line_code + diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last + + Gitlab::Git + .diff_line_code(file_path, diff_line.new_pos, diff_line.old_pos) + end + + # Returns a Hash that can be used to populate `notes.st_diff`, removing + # the need for requesting Git data for every diff note. + def diff_hash + { + diff: diff_hunk, + new_path: file_path, + old_path: file_path, + + # These fields are not displayed for LegacyDiffNote notes, so it + # doesn't really matter what we set them to. + a_mode: '100644', + b_mode: '100644', + new_file: false + } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/expose_attribute.rb b/lib/gitlab/github_import/representation/expose_attribute.rb new file mode 100644 index 00000000000..c3405759631 --- /dev/null +++ b/lib/gitlab/github_import/representation/expose_attribute.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module ExposeAttribute + extend ActiveSupport::Concern + + module ClassMethods + # Defines getter methods for the given attribute names. + # + # Example: + # + # expose_attribute :iid, :title + def expose_attribute(*names) + names.each do |name| + name = name.to_sym + + define_method(name) { attributes[name] } + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb new file mode 100644 index 00000000000..f3071b3e2b3 --- /dev/null +++ b/lib/gitlab/github_import/representation/issue.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Issue + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :iid, :title, :description, :milestone_number, + :created_at, :updated_at, :state, :assignees, + :label_names, :author + + # Builds an issue from a GitHub API response. + # + # issue - An instance of `Sawyer::Resource` containing the issue + # details. + def self.from_api_response(issue) + user = + if issue.user + Representation::User.from_api_response(issue.user) + end + + hash = { + iid: issue.number, + title: issue.title, + description: issue.body, + milestone_number: issue.milestone&.number, + state: issue.state == 'open' ? :opened : :closed, + assignees: issue.assignees.map do |u| + Representation::User.from_api_response(u) + end, + label_names: issue.labels.map(&:name), + author: user, + created_at: issue.created_at, + updated_at: issue.updated_at, + pull_request: issue.pull_request ? true : false + } + + new(hash) + end + + # Builds a new issue using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:state] = hash[:state].to_sym + hash[:assignees].map! { |u| Representation::User.from_json_hash(u) } + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A hash containing the raw issue details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def truncated_title + title.truncate(255) + end + + def labels? + label_names && label_names.any? + end + + def pull_request? + attributes[:pull_request] + end + + def issuable_type + pull_request? ? 'MergeRequest' : 'Issue' + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb new file mode 100644 index 00000000000..a68bc4c002f --- /dev/null +++ b/lib/gitlab/github_import/representation/note.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Note + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :noteable_id, :noteable_type, :author, :note, + :created_at, :updated_at, :github_id + + NOTEABLE_TYPE_REGEX = /\/(?(pull|issues))\/(?\d+)/i + + # Builds a note from a GitHub API response. + # + # note - An instance of `Sawyer::Resource` containing the note details. + def self.from_api_response(note) + matches = note.html_url.match(NOTEABLE_TYPE_REGEX) + + if !matches || !matches[:type] + raise( + ArgumentError, + "The note URL #{note.html_url.inspect} is not supported" + ) + end + + noteable_type = + if matches[:type] == 'pull' + 'MergeRequest' + else + 'Issue' + end + + user = Representation::User.from_api_response(note.user) if note.user + hash = { + noteable_type: noteable_type, + noteable_id: matches[:iid].to_i, + author: user, + note: note.body, + created_at: note.created_at, + updated_at: note.updated_at, + github_id: note.id + } + + new(hash) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + alias_method :issuable_type, :noteable_type + end + end + end +end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb new file mode 100644 index 00000000000..593b491a837 --- /dev/null +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class PullRequest + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :iid, :title, :description, :source_branch, + :source_branch_sha, :target_branch, :target_branch_sha, + :milestone_number, :author, :assignee, :created_at, + :updated_at, :merged_at, :source_repository_id, + :target_repository_id, :source_repository_owner + + # Builds a PR from a GitHub API response. + # + # issue - An instance of `Sawyer::Resource` containing the PR details. + def self.from_api_response(pr) + assignee = + if pr.assignee + Representation::User.from_api_response(pr.assignee) + end + + user = Representation::User.from_api_response(pr.user) if pr.user + hash = { + iid: pr.number, + title: pr.title, + description: pr.body, + source_branch: pr.head.ref, + target_branch: pr.base.ref, + source_branch_sha: pr.head.sha, + target_branch_sha: pr.base.sha, + source_repository_id: pr.head&.repo&.id, + target_repository_id: pr.base&.repo&.id, + source_repository_owner: pr.head&.user&.login, + state: pr.state == 'open' ? :opened : :closed, + milestone_number: pr.milestone&.number, + author: user, + assignee: assignee, + created_at: pr.created_at, + updated_at: pr.updated_at, + merged_at: pr.merged_at + } + + new(hash) + end + + # Builds a new PR using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:state] = hash[:state].to_sym + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + # Assignees are optional so we only convert it from a Hash if one was + # set. + hash[:assignee] &&= Representation::User + .from_json_hash(hash[:assignee]) + + new(hash) + end + + # attributes - A Hash containing the raw PR details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def truncated_title + title.truncate(255) + end + + # Returns a formatted source branch. + # + # For cross-project pull requests the branch name will be in the format + # `owner-name:branch-name`. + def formatted_source_branch + if cross_project? && source_repository_owner + "#{source_repository_owner}:#{source_branch}" + elsif source_branch == target_branch + # Sometimes the source and target branch are the same, but GitLab + # doesn't support this. This can happen when both the user and + # source repository have been deleted, and the PR was submitted from + # the fork's master branch. + "#{source_branch}-#{iid}" + else + source_branch + end + end + + def state + if merged_at + :merged + else + attributes[:state] + end + end + + def cross_project? + return true unless source_repository_id + + source_repository_id != target_repository_id + end + + def issuable_type + 'MergeRequest' + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/to_hash.rb b/lib/gitlab/github_import/representation/to_hash.rb new file mode 100644 index 00000000000..4a0f36ab8f0 --- /dev/null +++ b/lib/gitlab/github_import/representation/to_hash.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module ToHash + # Converts the current representation to a Hash. The keys of this Hash + # will be Symbols. + def to_hash + hash = {} + + attributes.each do |key, value| + hash[key] = convert_value_for_to_hash(value) + end + + hash + end + + def convert_value_for_to_hash(value) + if value.is_a?(Array) + value.map { |v| convert_value_for_to_hash(v) } + elsif value.respond_to?(:to_hash) + value.to_hash + else + value + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb new file mode 100644 index 00000000000..e00dcfca33d --- /dev/null +++ b/lib/gitlab/github_import/representation/user.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class User + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :login + + # Builds a user from a GitHub API response. + # + # user - An instance of `Sawyer::Resource` containing the user details. + def self.from_api_response(user) + new(id: user.id, login: user.login) + end + + # Builds a user using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the user details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + end + end + end +end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb new file mode 100644 index 00000000000..4f7324536a0 --- /dev/null +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # The SequentialImporter imports a GitHub project in a single thread, + # without using Sidekiq. This makes it useful for testing purposes as well + # as Rake tasks, but it should be avoided for anything else in favour of the + # parallel importer. + class SequentialImporter + attr_reader :project, :client + + SEQUENTIAL_IMPORTERS = [ + Importer::LabelsImporter, + Importer::MilestonesImporter, + Importer::ReleasesImporter + ].freeze + + PARALLEL_IMPORTERS = [ + Importer::PullRequestsImporter, + Importer::IssuesImporter, + Importer::DiffNotesImporter, + Importer::NotesImporter + ].freeze + + # project - The project to import the data into. + # token - The token to use for the GitHub API. + def initialize(project, token: nil) + @project = project + @client = GithubImport + .new_client_for(project, token: token, parallel: false) + end + + def execute + Importer::RepositoryImporter.new(project, client).execute + + SEQUENTIAL_IMPORTERS.each do |klass| + klass.new(project, client).execute + end + + PARALLEL_IMPORTERS.each do |klass| + klass.new(project, client, parallel: false).execute + end + + project.repository.after_import + + true + end + end + end +end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb new file mode 100644 index 00000000000..be1259662a7 --- /dev/null +++ b/lib/gitlab/github_import/user_finder.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Class that can be used for finding a GitLab user ID based on a GitHub user + # ID or username. + # + # Any found user IDs are cached in Redis to reduce the number of SQL queries + # executed over time. Valid keys are refreshed upon access so frequently + # used keys stick around. + # + # Lookups are cached even if no ID was found to remove the need for querying + # the database when most queries are not going to return results anyway. + class UserFinder + attr_reader :project, :client + + # The base cache key to use for caching user IDs for a given GitHub user + # ID. + ID_CACHE_KEY = 'github-import/user-finder/user-id/%s'.freeze + + # The base cache key to use for caching user IDs for a given GitHub email + # address. + ID_FOR_EMAIL_CACHE_KEY = + 'github-import/user-finder/id-for-email/%s'.freeze + + # The base cache key to use for caching the Email addresses of GitHub + # usernames. + EMAIL_FOR_USERNAME_CACHE_KEY = + 'github-import/user-finder/email-for-username/%s'.freeze + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + end + + # Returns the GitLab user ID of an object's author. + # + # If the object has no author ID we'll use the ID of the GitLab ghost + # user. + def author_id_for(object) + id = + if object&.author + user_id_for(object.author) + else + GithubImport.ghost_user_id + end + + if id + [id, true] + else + [project.creator_id, false] + end + end + + # Returns the GitLab user ID of an issuable's assignee. + def assignee_id_for(issuable) + user_id_for(issuable.assignee) if issuable.assignee + end + + # Returns the GitLab user ID for a GitHub user. + # + # user - An instance of `Gitlab::GithubImport::Representation::User`. + def user_id_for(user) + find(user.id, user.login) + end + + # Returns the GitLab ID for the given GitHub ID or username. + # + # id - The ID of the GitHub user. + # username - The username of the GitHub user. + def find(id, username) + email = email_for_github_username(username) + cached, found_id = find_from_cache(id, email) + + return found_id if found_id + + # We only want to query the database if necessary. If previous lookups + # didn't yield a user ID we won't query the database again until the + # keys expire. + find_id_from_database(id, email) unless cached + end + + # Finds a user ID from the cache for a given GitHub ID or Email. + def find_from_cache(id, email = nil) + id_exists, id_for_github_id = cached_id_for_github_id(id) + + return [id_exists, id_for_github_id] if id_for_github_id + + # Just in case no Email address could be retrieved (for whatever reason) + return [false] unless email + + cached_id_for_github_email(email) + end + + # Finds a GitLab user ID from the database for a given GitHub user ID or + # Email. + def find_id_from_database(id, email) + id_for_github_id(id) || id_for_github_email(email) + end + + def email_for_github_username(username) + cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username + email = Caching.read(cache_key) + + unless email + user = client.user(username) + email = Caching.write(cache_key, user.email) if user + end + + email + end + + def cached_id_for_github_id(id) + read_id_from_cache(ID_CACHE_KEY % id) + end + + def cached_id_for_github_email(email) + read_id_from_cache(ID_FOR_EMAIL_CACHE_KEY % email) + end + + # Queries and caches the GitLab user ID for a GitHub user ID, if one was + # found. + def id_for_github_id(id) + gitlab_id = query_id_for_github_id(id) || nil + + Caching.write(ID_CACHE_KEY % id, gitlab_id) + end + + # Queries and caches the GitLab user ID for a GitHub email, if one was + # found. + def id_for_github_email(email) + gitlab_id = query_id_for_github_email(email) || nil + + Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) + end + + def query_id_for_github_id(id) + User.for_github_id(id).pluck(:id).first + end + + def query_id_for_github_email(email) + User.by_any_email(email).pluck(:id).first + end + + # Reads an ID from the cache. + # + # The return value is an Array with two values: + # + # 1. A boolean indicating if the key was present or not. + # 2. The ID as an Integer, or nil in case no ID could be found. + def read_id_from_cache(key) + value = Caching.read(key) + exists = !value.nil? + number = value.to_i + + # The cache key may be empty to indicate a previously looked up user for + # which we couldn't find an ID. + [exists, number.positive? ? number : nil] + end + end + end +end diff --git a/lib/gitlab/github_import/user_formatter.rb b/lib/gitlab/github_import/user_formatter.rb deleted file mode 100644 index 04c2964da20..00000000000 --- a/lib/gitlab/github_import/user_formatter.rb +++ /dev/null @@ -1,45 +0,0 @@ -module Gitlab - module GithubImport - class UserFormatter - attr_reader :client, :raw - - delegate :id, :login, to: :raw, allow_nil: true - - def initialize(client, raw) - @client = client - @raw = raw - end - - def gitlab_id - return @gitlab_id if defined?(@gitlab_id) - - @gitlab_id = find_by_external_uid || find_by_email - end - - private - - def email - @email ||= client.user(raw.login).try(:email) - end - - def find_by_email - return nil unless email - - User.find_by_any_email(email) - .try(:id) - end - - def find_by_external_uid - return nil unless id - - identities = ::Identity.arel_table - - User.select(:id) - .joins(:identities).where(identities[:provider].eq(:github) - .and(identities[:extern_uid].eq(id))) - .first - .try(:id) - end - end - end -end diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb deleted file mode 100644 index ca8d96f5650..00000000000 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Gitlab - module GithubImport - class WikiFormatter - attr_reader :project - - def initialize(project) - @project = project - end - - def disk_path - project.wiki.disk_path - end - - def import_url - project.import_url.sub(/\.git\z/, ".wiki.git") - end - end - end -end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 5404dc11a87..c730fefcffe 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -15,7 +15,7 @@ module Gitlab ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repo by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), - ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer) ].freeze class << self diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 4d6bbda15f3..f654508c391 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -19,11 +19,13 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end - attr_reader :key, :jobs_remaining, :finished + attr_reader :key, :finished + attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for - def initialize(jobs_remaining) - @key = "gitlab:job_waiter:#{SecureRandom.uuid}" + # key - The key of this waiter. + def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}") + @key = key @jobs_remaining = jobs_remaining @finished = [] end diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb new file mode 100644 index 00000000000..2f07fde406c --- /dev/null +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -0,0 +1,26 @@ +module Gitlab + module LegacyGithubImport + class BaseFormatter + attr_reader :client, :formatter, :project, :raw_data + + def initialize(project, raw_data, client = nil) + @project = project + @raw_data = raw_data + @client = client + @formatter = Gitlab::ImportFormatter.new + end + + def create! + association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend + + association.find_or_create_by!(find_condition) do |record| + record.attributes = attributes + end + end + + def url + raw_data.url || '' + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/branch_formatter.rb b/lib/gitlab/legacy_github_import/branch_formatter.rb new file mode 100644 index 00000000000..80fe1d67209 --- /dev/null +++ b/lib/gitlab/legacy_github_import/branch_formatter.rb @@ -0,0 +1,37 @@ +module Gitlab + module LegacyGithubImport + class BranchFormatter < BaseFormatter + delegate :repo, :sha, :ref, to: :raw_data + + def exists? + branch_exists? && commit_exists? + end + + def valid? + sha.present? && ref.present? + end + + def user + raw_data.user&.login || 'unknown' + end + + def short_sha + Commit.truncate_sha(sha) + end + + private + + def branch_exists? + project.repository.branch_exists?(ref) + end + + def commit_exists? + project.repository.branch_names_contains(sha).include?(ref) + end + + def short_id + sha.to_s[0..7] + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb new file mode 100644 index 00000000000..53c910d44bd --- /dev/null +++ b/lib/gitlab/legacy_github_import/client.rb @@ -0,0 +1,148 @@ +module Gitlab + module LegacyGithubImport + class Client + GITHUB_SAFE_REMAINING_REQUESTS = 100 + GITHUB_SAFE_SLEEP_TIME = 500 + + attr_reader :access_token, :host, :api_version + + def initialize(access_token, host: nil, api_version: 'v3') + @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version + @users = {} + + if access_token + ::Octokit.auto_paginate = false + end + end + + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: api_endpoint, + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end + + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' + end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) + end + + def authorize_url(redirect_uri) + client.auth_code.authorize_url({ + redirect_uri: redirect_uri, + scope: "repo, user, user:email" + }) + end + + def get_token(code) + client.auth_code.get_token(code).token + end + + def method_missing(method, *args, &block) + if api.respond_to?(method) + request(method, *args, &block) + else + super(method, *args, &block) + end + end + + def respond_to?(method) + api.respond_to?(method) || super + end + + def user(login) + return nil unless login.present? + return @users[login] if @users.key?(login) + + @users[login] = api.user(login) + end + + private + + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } + end + + def github_options + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end + end + + def rate_limit + api.rate_limit! + # GitHub Rate Limit API returns 404 when the rate limit is + # disabled. In this case we just want to return gracefully + # instead of spitting out an error. + rescue Octokit::NotFound + nil + end + + def has_rate_limit? + return @has_rate_limit if defined?(@has_rate_limit) + + @has_rate_limit = rate_limit.present? + end + + def rate_limit_exceed? + has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + end + + def rate_limit_sleep_time + rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + end + + def request(method, *args, &block) + sleep rate_limit_sleep_time if rate_limit_exceed? + + data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend + return data unless data.is_a?(Array) + + last_response = api.last_response + + if block_given? + yield data + # api.last_response could change while we're yielding (e.g. fetching labels for each PR) + # so we cache our own last response + each_response_page(last_response, &block) + else + each_response_page(last_response) { |page| data.concat(page) } + data + end + end + + def each_response_page(last_response) + while last_response.rels[:next] + sleep rate_limit_sleep_time if rate_limit_exceed? + last_response = last_response.rels[:next].get + yield last_response.data if last_response.data.is_a?(Array) + end + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb new file mode 100644 index 00000000000..d2c7a8ae9f4 --- /dev/null +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -0,0 +1,69 @@ +module Gitlab + module LegacyGithubImport + class CommentFormatter < BaseFormatter + attr_writer :author_id + + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + type: type, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + @author ||= UserFormatter.new(client, raw_data.user) + end + + def author_id + author.gitlab_id || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + return unless on_diff? + + parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) + generate_line_code(parsed_lines.to_a.last) + end + + def generate_line_code(line) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) + end + + def on_diff? + diff_hunk.present? + end + + def diff_hunk + raw_data.diff_hunk + end + + def file_path + raw_data.path + end + + def note + if author.gitlab_id + body + else + formatter.author_line(author.login) + body + end + end + + def type + 'LegacyDiffNote' if on_diff? + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb new file mode 100644 index 00000000000..12c968805f5 --- /dev/null +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -0,0 +1,329 @@ +module Gitlab + module LegacyGithubImport + class Importer + include Gitlab::ShellAdapter + + attr_reader :errors, :project, :repo, :repo_url + + def initialize(project) + @project = project + @repo = project.import_source + @repo_url = project.import_url + @errors = [] + @labels = {} + end + + def client + return @client if defined?(@client) + unless credentials + raise Projects::ImportService::Error, + "Unable to find project import data credentials for project ID: #{@project.id}" + end + + opts = {} + # Gitea plan to be GitHub compliant + if project.gitea_import? + uri = URI.parse(project.import_url) + host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') + opts = { + host: host, + api_version: 'v1' + } + end + + @client = Client.new(credentials[:user], opts) + end + + def execute + # The ordering of importing is important here due to the way GitHub structures their data + # 1. Labels are required by other items while not having a dependency on anything else + # so need to be first + # 2. Pull requests must come before issues. Every pull request is also an issue but not + # all issues are pull requests. Only the issue entity has labels defined in GitHub. GitLab + # doesn't structure data like this so we need to make sure that we've created the MRs + # before we attempt to add the labels defined in the GitHub issue for the related, already + # imported, pull request + import_labels + import_milestones + import_pull_requests + import_issues + import_comments(:issues) + import_comments(:pull_requests) + import_wiki + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + + handle_errors + + true + end + + private + + def credentials + return @credentials if defined?(@credentials) + + @credentials = project.import_data ? project.import_data.credentials : nil + end + + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def import_labels + fetch_resources(:labels, repo, per_page: 100) do |labels| + labels.each do |raw| + begin + gh_label = LabelFormatter.new(project, raw) + gh_label.create! + rescue => e + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } + end + end + end + + cache_labels! + end + + def import_milestones + fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| + milestones.each do |raw| + begin + gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone.create! + rescue => e + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } + end + end + end + end + + def import_issues + fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| + issues.each do |raw| + gh_issue = IssueFormatter.new(project, raw, client) + + begin + issuable = + if gh_issue.pull_request? + MergeRequest.find_by(target_project_id: project.id, iid: gh_issue.number) + else + gh_issue.create! + end + + apply_labels(issuable, raw) + rescue => e + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } + end + end + end + end + + def import_pull_requests + fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| + pull_requests.each do |raw| + gh_pull_request = PullRequestFormatter.new(project, raw, client) + + next unless gh_pull_request.valid? + + begin + restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists? + restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists? + + merge_request = gh_pull_request.create! + + # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage + if project.gitea_import? + apply_labels(merge_request, raw) + end + rescue => e + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } + ensure + clean_up_restored_branches(gh_pull_request) + end + end + end + + project.repository.after_remove_branch + end + + def restore_source_branch(pull_request) + project.repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) + end + + def restore_target_branch(pull_request) + project.repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) + end + + def remove_branch(name) + project.repository.delete_branch(name) + rescue Gitlab::Git::Repository::DeleteBranchFailed + errors << { type: :remove_branch, name: name } + end + + def clean_up_restored_branches(pull_request) + return if pull_request.opened? + + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? + remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? + end + + def apply_labels(issuable, raw) + return unless raw.labels.count > 0 + + label_ids = raw.labels + .map { |attrs| @labels[attrs.name] } + .compact + + issuable.update_attribute(:label_ids, label_ids) + end + + def import_comments(issuable_type) + resource_type = "#{issuable_type}_comments".to_sym + + # Two notes here: + # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note, + # compare it against every comment in the current imported page until we find match, and that's where start importing + # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns + # only comments on diffs, so select last note not based on noteable_type but on line_code + line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL' + last_note = project.notes.where("line_code IS #{line_code_is}").last + + fetch_resources(resource_type, repo, per_page: 100) do |comments| + if last_note + discard_inserted_comments(comments, last_note) + last_note = nil + end + + create_comments(comments) + end + end + + def create_comments(comments) + ActiveRecord::Base.no_touching do + comments.each do |raw| + begin + comment = CommentFormatter.new(project, raw, client) + + # GH does not return info about comment's parent, so we guess it by checking its URL! + *_, parent, iid = URI(raw.html_url).path.split('/') + + issuable = if parent == 'issues' + Issue.find_by(project_id: project.id, iid: iid) + else + MergeRequest.find_by(target_project_id: project.id, iid: iid) + end + + next unless issuable + + issuable.notes.create!(comment.attributes) + rescue => e + errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end + end + + def discard_inserted_comments(comments, last_note) + last_note_attrs = nil + + cut_off_index = comments.find_index do |raw| + comment = CommentFormatter.new(project, raw) + comment_attrs = comment.attributes + last_note_attrs ||= last_note.slice(*comment_attrs.keys) + + comment_attrs.with_indifferent_access == last_note_attrs + end + + # No matching resource in the collection, which means we got halted right on the end of the last page, so all good + return unless cut_off_index + + # Otherwise, remove the resources we've already inserted + comments.shift(cut_off_index + 1) + end + + def import_wiki + unless project.wiki.repository_exists? + wiki = WikiFormatter.new(project) + gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) + end + rescue Gitlab::Shell::Error => e + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + errors << { type: :wiki, errors: e.message } + end + end + + def import_releases + fetch_resources(:releases, repo, per_page: 100) do |releases| + releases.each do |raw| + begin + gh_release = ReleaseFormatter.new(project, raw) + gh_release.create! if gh_release.valid? + rescue => e + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } + end + end + end + end + + def cache_labels! + project.labels.select(:id, :title).find_each do |label| + @labels[label.title] = label.id + end + end + + def fetch_resources(resource_type, *opts) + return if imported?(resource_type) + + opts.last[:page] = current_page(resource_type) + + client.public_send(resource_type, *opts) do |resources| # rubocop:disable GitlabSecurity/PublicSend + yield resources + increment_page(resource_type) + end + + imported!(resource_type) + end + + def imported?(resource_type) + Rails.cache.read("#{cache_key_prefix}:#{resource_type}:imported") + end + + def imported!(resource_type) + Rails.cache.write("#{cache_key_prefix}:#{resource_type}:imported", true, ex: 1.day) + end + + def increment_page(resource_type) + key = "#{cache_key_prefix}:#{resource_type}:current-page" + + # Rails.cache.increment calls INCRBY directly on the value stored under the key, which is + # a serialized ActiveSupport::Cache::Entry, so it will return an error by Redis, hence this ugly work-around + page = Rails.cache.read(key) + page += 1 + Rails.cache.write(key, page) + + page + end + + def current_page(resource_type) + Rails.cache.fetch("#{cache_key_prefix}:#{resource_type}:current-page", ex: 1.day) { 1 } + end + + def cache_key_prefix + @cache_key_prefix ||= "github-import:#{project.id}" + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb new file mode 100644 index 00000000000..de55382d3ad --- /dev/null +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -0,0 +1,66 @@ +module Gitlab + module LegacyGithubImport + class IssuableFormatter < BaseFormatter + attr_writer :assignee_id, :author_id + + def project_association + raise NotImplementedError + end + + delegate :number, to: :raw_data + + def find_condition + { iid: number } + end + + private + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def assigned? + raw_data.assignee.present? + end + + def author + @author ||= UserFormatter.new(client, raw_data.user) + end + + def author_id + @author_id ||= author.gitlab_id || project.creator_id + end + + def assignee + if assigned? + @assignee ||= UserFormatter.new(client, raw_data.assignee) + end + end + + def assignee_id + return @assignee_id if defined?(@assignee_id) + + @assignee_id = assignee.try(:gitlab_id) + end + + def body + raw_data.body || "" + end + + def description + if author.gitlab_id + body + else + formatter.author_line(author.login) + body + end + end + + def milestone + if raw_data.milestone.present? + milestone = MilestoneFormatter.new(project, raw_data.milestone) + project.milestones.find_by(milestone.find_condition) + end + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb new file mode 100644 index 00000000000..4c8825ccf19 --- /dev/null +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -0,0 +1,32 @@ +module Gitlab + module LegacyGithubImport + class IssueFormatter < IssuableFormatter + def attributes + { + iid: number, + project: project, + milestone: milestone, + title: raw_data.title, + description: description, + state: state, + author_id: author_id, + assignee_ids: Array(assignee_id), + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + def has_comments? + raw_data.comments > 0 + end + + def project_association + :issues + end + + def pull_request? + raw_data.pull_request.present? + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb new file mode 100644 index 00000000000..c3eed12e739 --- /dev/null +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -0,0 +1,37 @@ +module Gitlab + module LegacyGithubImport + class LabelFormatter < BaseFormatter + def attributes + { + project: project, + title: title, + color: color + } + end + + def project_association + :labels + end + + def create! + params = attributes.except(:project) + service = ::Labels::FindOrCreateService.new(nil, project, params) + label = service.execute(skip_authorization: true) + + raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? + + label + end + + private + + def color + "##{raw_data.color}" + end + + def title + raw_data.name + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb new file mode 100644 index 00000000000..a565294384d --- /dev/null +++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb @@ -0,0 +1,40 @@ +module Gitlab + module LegacyGithubImport + class MilestoneFormatter < BaseFormatter + def attributes + { + iid: number, + project: project, + title: raw_data.title, + description: raw_data.description, + due_date: raw_data.due_on, + state: state, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + def project_association + :milestones + end + + def find_condition + { iid: number } + end + + def number + if project.gitea_import? + raw_data.id + else + raw_data.number + end + end + + private + + def state + raw_data.state == 'closed' ? 'closed' : 'active' + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb new file mode 100644 index 00000000000..41e7eac4d08 --- /dev/null +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -0,0 +1,52 @@ +module Gitlab + module LegacyGithubImport + class ProjectCreator + include Gitlab::CurrentSettings + + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type + + def initialize(repo, name, namespace, current_user, session_data, type: 'github') + @repo = repo + @name = name + @namespace = namespace + @current_user = current_user + @session_data = session_data + @type = type + end + + def execute + ::Projects::CreateService.new( + current_user, + name: name, + path: name, + description: repo.description, + namespace_id: namespace.id, + visibility_level: visibility_level, + import_type: type, + import_source: repo.full_name, + import_url: import_url, + skip_wiki: skip_wiki + ).execute + end + + private + + def import_url + repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") + end + + def visibility_level + repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + end + + # + # If the GitHub project repository has wiki, we should not create the + # default wiki. Otherwise the GitHub importer will fail because the wiki + # repository already exist. + # + def skip_wiki + repo.has_wiki? + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb new file mode 100644 index 00000000000..94c2e99066a --- /dev/null +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -0,0 +1,90 @@ +module Gitlab + module LegacyGithubImport + class PullRequestFormatter < IssuableFormatter + delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true + + def attributes + { + iid: number, + title: raw_data.title, + description: description, + source_project: source_branch_project, + source_branch: source_branch_name, + source_branch_sha: source_branch_sha, + target_project: target_branch_project, + target_branch: target_branch_name, + target_branch_sha: target_branch_sha, + state: state, + milestone: milestone, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at, + imported: true + } + end + + def project_association + :merge_requests + end + + def valid? + source_branch.valid? && target_branch.valid? + end + + def source_branch + @source_branch ||= BranchFormatter.new(project, raw_data.head) + end + + def source_branch_name + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed + else + source_branch_ref + end + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{number}/#{source_branch_user}/#{source_branch_ref}" + end + + def source_branch_exists? + !cross_project? && source_branch.exists? + end + + def target_branch + @target_branch ||= BranchFormatter.new(project, raw_data.base) + end + + def target_branch_name + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{number}/#{target_branch_user}/#{target_branch_ref}" + end + + def cross_project? + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id + end + + def opened? + state == 'opened' + end + + private + + def state + if raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + else + super + end + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb new file mode 100644 index 00000000000..3ed9d4f76da --- /dev/null +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -0,0 +1,27 @@ +module Gitlab + module LegacyGithubImport + class ReleaseFormatter < BaseFormatter + def attributes + { + project: project, + tag: raw_data.tag_name, + description: raw_data.body, + created_at: raw_data.created_at, + updated_at: raw_data.created_at + } + end + + def project_association + :releases + end + + def find_condition + { tag: raw_data.tag_name } + end + + def valid? + !raw_data.draft + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb new file mode 100644 index 00000000000..6d8055622f1 --- /dev/null +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module LegacyGithubImport + class UserFormatter + attr_reader :client, :raw + + delegate :id, :login, to: :raw, allow_nil: true + + def initialize(client, raw) + @client = client + @raw = raw + end + + def gitlab_id + return @gitlab_id if defined?(@gitlab_id) + + @gitlab_id = find_by_external_uid || find_by_email + end + + private + + def email + @email ||= client.user(raw.login).try(:email) + end + + def find_by_email + return nil unless email + + User.find_by_any_email(email) + .try(:id) + end + + def find_by_external_uid + return nil unless id + + identities = ::Identity.arel_table + + User.select(:id) + .joins(:identities).where(identities[:provider].eq(:github) + .and(identities[:extern_uid].eq(id))) + .first + .try(:id) + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/wiki_formatter.rb b/lib/gitlab/legacy_github_import/wiki_formatter.rb new file mode 100644 index 00000000000..27f45875c7c --- /dev/null +++ b/lib/gitlab/legacy_github_import/wiki_formatter.rb @@ -0,0 +1,19 @@ +module Gitlab + module LegacyGithubImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def disk_path + project.wiki.disk_path + end + + def import_url + project.import_url.sub(/\.git\z/, ".wiki.git") + end + end + end +end -- cgit v1.2.1 From 6e242e82237ad2cf362098f3f42f4a9dd1a4ad27 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 18 Oct 2017 21:46:05 +0200 Subject: Replace old GH importer with the parallel importer --- lib/github/client.rb | 54 ----- lib/github/collection.rb | 29 --- lib/github/error.rb | 3 - lib/github/import.rb | 377 ------------------------------ lib/github/import/issue.rb | 13 -- lib/github/import/legacy_diff_note.rb | 12 - lib/github/import/merge_request.rb | 13 -- lib/github/import/note.rb | 13 -- lib/github/rate_limit.rb | 27 --- lib/github/repositories.rb | 19 -- lib/github/representation/base.rb | 30 --- lib/github/representation/branch.rb | 55 ----- lib/github/representation/comment.rb | 42 ---- lib/github/representation/issuable.rb | 37 --- lib/github/representation/issue.rb | 27 --- lib/github/representation/label.rb | 13 -- lib/github/representation/milestone.rb | 25 -- lib/github/representation/pull_request.rb | 71 ------ lib/github/representation/release.rb | 17 -- lib/github/representation/repo.rb | 6 - lib/github/representation/user.rb | 15 -- lib/github/response.rb | 25 -- lib/github/user.rb | 24 -- lib/gitlab/import_sources.rb | 2 +- lib/tasks/import.rake | 6 +- 25 files changed, 5 insertions(+), 950 deletions(-) delete mode 100644 lib/github/client.rb delete mode 100644 lib/github/collection.rb delete mode 100644 lib/github/error.rb delete mode 100644 lib/github/import.rb delete mode 100644 lib/github/import/issue.rb delete mode 100644 lib/github/import/legacy_diff_note.rb delete mode 100644 lib/github/import/merge_request.rb delete mode 100644 lib/github/import/note.rb delete mode 100644 lib/github/rate_limit.rb delete mode 100644 lib/github/repositories.rb delete mode 100644 lib/github/representation/base.rb delete mode 100644 lib/github/representation/branch.rb delete mode 100644 lib/github/representation/comment.rb delete mode 100644 lib/github/representation/issuable.rb delete mode 100644 lib/github/representation/issue.rb delete mode 100644 lib/github/representation/label.rb delete mode 100644 lib/github/representation/milestone.rb delete mode 100644 lib/github/representation/pull_request.rb delete mode 100644 lib/github/representation/release.rb delete mode 100644 lib/github/representation/repo.rb delete mode 100644 lib/github/representation/user.rb delete mode 100644 lib/github/response.rb delete mode 100644 lib/github/user.rb (limited to 'lib') diff --git a/lib/github/client.rb b/lib/github/client.rb deleted file mode 100644 index 29bd9c1f39e..00000000000 --- a/lib/github/client.rb +++ /dev/null @@ -1,54 +0,0 @@ -module Github - class Client - TIMEOUT = 60 - DEFAULT_PER_PAGE = 100 - - attr_reader :connection, :rate_limit - - def initialize(options) - @connection = Faraday.new(url: options.fetch(:url, root_endpoint)) do |faraday| - faraday.options.open_timeout = options.fetch(:timeout, TIMEOUT) - faraday.options.timeout = options.fetch(:timeout, TIMEOUT) - faraday.authorization 'token', options.fetch(:token) - faraday.adapter :net_http - faraday.ssl.verify = verify_ssl - end - - @rate_limit = RateLimit.new(connection) - end - - def get(url, query = {}) - exceed, reset_in = rate_limit.get - sleep reset_in if exceed - - Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query))) - end - - private - - def root_endpoint - custom_endpoint || github_endpoint - end - - def custom_endpoint - github_omniauth_provider.dig('args', 'client_options', 'site') - end - - def verify_ssl - # If there is no config, we're connecting to github.com - # and we should verify ssl. - github_omniauth_provider.fetch('verify_ssl', true) - end - - def github_endpoint - OmniAuth::Strategies::GitHub.default_options[:client_options][:site] - end - - def github_omniauth_provider - @github_omniauth_provider ||= - Gitlab.config.omniauth.providers - .find { |provider| provider.name == 'github' } - .to_h - end - end -end diff --git a/lib/github/collection.rb b/lib/github/collection.rb deleted file mode 100644 index 014b2038c4b..00000000000 --- a/lib/github/collection.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Github - class Collection - attr_reader :options - - def initialize(options) - @options = options - end - - def fetch(url, query = {}) - return [] if url.blank? - - Enumerator.new do |yielder| - loop do - response = client.get(url, query) - response.body.each { |item| yielder << item } - - raise StopIteration unless response.rels.key?(:next) - url = response.rels[:next] - end - end.lazy - end - - private - - def client - @client ||= Github::Client.new(options) - end - end -end diff --git a/lib/github/error.rb b/lib/github/error.rb deleted file mode 100644 index 66d7afaa787..00000000000 --- a/lib/github/error.rb +++ /dev/null @@ -1,3 +0,0 @@ -module Github - RepositoryFetchError = Class.new(StandardError) -end diff --git a/lib/github/import.rb b/lib/github/import.rb deleted file mode 100644 index fef63dd7168..00000000000 --- a/lib/github/import.rb +++ /dev/null @@ -1,377 +0,0 @@ -require_relative 'error' -require_relative 'import/issue' -require_relative 'import/legacy_diff_note' -require_relative 'import/merge_request' -require_relative 'import/note' - -module Github - class Import - include Gitlab::ShellAdapter - - attr_reader :project, :repository, :repo, :repo_url, :wiki_url, - :options, :errors, :cached, :verbose, :last_fetched_at - - def initialize(project, options = {}) - @project = project - @repository = project.repository - @repo = project.import_source - @repo_url = project.import_url - @wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') - @options = options.reverse_merge(token: project.import_data&.credentials&.fetch(:user)) - @verbose = options.fetch(:verbose, false) - @cached = Hash.new { |hash, key| hash[key] = Hash.new } - @errors = [] - @last_fetched_at = nil - end - - # rubocop: disable Rails/Output - def execute - puts 'Fetching repository...'.color(:aqua) if verbose - setup_and_fetch_repository - puts 'Fetching labels...'.color(:aqua) if verbose - fetch_labels - puts 'Fetching milestones...'.color(:aqua) if verbose - fetch_milestones - puts 'Fetching pull requests...'.color(:aqua) if verbose - fetch_pull_requests - puts 'Fetching issues...'.color(:aqua) if verbose - fetch_issues - puts 'Fetching releases...'.color(:aqua) if verbose - fetch_releases - puts 'Cloning wiki repository...'.color(:aqua) if verbose - fetch_wiki_repository - puts 'Expiring repository cache...'.color(:aqua) if verbose - expire_repository_cache - - errors.empty? - rescue Github::RepositoryFetchError - expire_repository_cache - false - ensure - keep_track_of_errors - end - - private - - def setup_and_fetch_repository - begin - project.ensure_repository - project.repository.add_remote('github', repo_url) - project.repository.set_import_remote_as_mirror('github') - project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') - fetch_remote(forced: true) - rescue Gitlab::Git::Repository::NoRepository, - Gitlab::Git::RepositoryMirroring::RemoteError, - Gitlab::Shell::Error => e - error(:project, repo_url, e.message) - raise Github::RepositoryFetchError - end - end - - def fetch_remote(forced: false) - @last_fetched_at = Time.now - project.repository.fetch_remote('github', forced: forced) - end - - def fetch_wiki_repository - return if project.wiki.repository_exists? - - wiki_path = project.wiki.disk_path - gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) - rescue Gitlab::Shell::Error => e - # GitHub error message when the wiki repo has not been created, - # this means that repo has wiki enabled, but have no pages. So, - # we can skip the import. - if e.message !~ /repository not exported/ - error(:wiki, wiki_url, e.message) - end - end - - def fetch_labels - url = "/repos/#{repo}/labels" - - while url - response = Github::Client.new(options).get(url) - - response.body.each do |raw| - begin - representation = Github::Representation::Label.new(raw) - - label = project.labels.find_or_create_by!(title: representation.title) do |label| - label.color = representation.color - end - - cached[:label_ids][representation.title] = label.id - rescue => e - error(:label, representation.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_milestones - url = "/repos/#{repo}/milestones" - - while url - response = Github::Client.new(options).get(url, state: :all) - - response.body.each do |raw| - begin - milestone = Github::Representation::Milestone.new(raw) - next if project.milestones.where(iid: milestone.iid).exists? - - project.milestones.create!( - iid: milestone.iid, - title: milestone.title, - description: milestone.description, - due_date: milestone.due_date, - state: milestone.state, - created_at: milestone.created_at, - updated_at: milestone.updated_at - ) - rescue => e - error(:milestone, milestone.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_pull_requests - url = "/repos/#{repo}/pulls" - - while url - response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - - response.body.each do |raw| - pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) - merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) - next unless merge_request.new_record? && pull_request.valid? - - begin - # If the PR has been created/updated after we last fetched the - # remote, we fetch again to get the up-to-date refs. - fetch_remote if pull_request.updated_at > last_fetched_at - - author_id = user_id(pull_request.author, project.creator_id) - description = format_description(pull_request.description, pull_request.author) - - merge_request.attributes = { - iid: pull_request.iid, - title: pull_request.title, - description: description, - source_project: pull_request.source_project, - source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, - target_project: pull_request.target_project, - target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, - state: pull_request.state, - milestone_id: milestone_id(pull_request.milestone), - author_id: author_id, - assignee_id: user_id(pull_request.assignee), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - } - - merge_request.save!(validate: false) - merge_request.merge_request_diffs.create - - review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" - fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) - rescue => e - error(:pull_request, pull_request.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_issues - url = "/repos/#{repo}/issues" - - while url - response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - - response.body.each { |raw| populate_issue(raw) } - - url = response.rels[:next] - end - end - - def populate_issue(raw) - representation = Github::Representation::Issue.new(raw, options) - - begin - # Every pull request is an issue, but not every issue - # is a pull request. For this reason, "shared" actions - # for both features, like manipulating assignees, labels - # and milestones, are provided within the Issues API. - if representation.pull_request? - return unless representation.labels? || representation.comments? - - merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - - if representation.labels? - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) - end - - fetch_comments_conditionally(merge_request, representation) - else - return if Issue.exists?(iid: representation.iid, project_id: project.id) - - author_id = user_id(representation.author, project.creator_id) - issue = Issue.new - issue.iid = representation.iid - issue.project_id = project.id - issue.title = representation.title - issue.description = format_description(representation.description, representation.author) - issue.state = representation.state - issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = author_id - issue.created_at = representation.created_at - issue.updated_at = representation.updated_at - issue.save!(validate: false) - - issue.update( - label_ids: label_ids(representation.labels), - assignee_ids: assignee_ids(representation.assignees)) - - fetch_comments_conditionally(issue, representation) - end - rescue => e - error(:issue, representation.url, e.message) - end - end - - def fetch_comments_conditionally(issuable, representation) - if representation.comments? - comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments" - fetch_comments(issuable, :comment, comments_url) - end - end - - def fetch_comments(noteable, type, url, klass = Note) - while url - comments = Github::Client.new(options).get(url) - - ActiveRecord::Base.no_touching do - comments.body.each do |raw| - begin - representation = Github::Representation::Comment.new(raw, options) - author_id = user_id(representation.author, project.creator_id) - - note = klass.new - note.project_id = project.id - note.noteable = noteable - note.note = format_description(representation.note, representation.author) - note.commit_id = representation.commit_id - note.line_code = representation.line_code - note.author_id = author_id - note.created_at = representation.created_at - note.updated_at = representation.updated_at - note.save!(validate: false) - rescue => e - error(type, representation.url, e.message) - end - end - end - - url = comments.rels[:next] - end - end - - def fetch_releases - url = "/repos/#{repo}/releases" - - while url - response = Github::Client.new(options).get(url) - - response.body.each do |raw| - representation = Github::Representation::Release.new(raw) - next unless representation.valid? - - release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) - next unless release.new_record? - - begin - release.description = representation.description - release.created_at = representation.created_at - release.updated_at = representation.updated_at - release.save!(validate: false) - rescue => e - error(:release, representation.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def label_ids(labels) - labels.map { |label| cached[:label_ids][label.title] }.compact - end - - def assignee_ids(assignees) - assignees.map { |assignee| user_id(assignee) }.compact - end - - def milestone_id(milestone) - return unless milestone.present? - - project.milestones.select(:id).find_by(iid: milestone.iid)&.id - end - - def user_id(user, fallback_id = nil) - return unless user.present? - return cached[:user_ids][user.id] if cached[:user_ids][user.id].present? - - gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) - - cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? - cached[:user_ids][user.id] = gitlab_user_id || fallback_id - end - - def user_id_by_email(email) - return nil unless email - - ::User.find_by_any_email(email)&.id - end - - def user_id_by_external_uid(id) - return nil unless id - - ::User.select(:id) - .joins(:identities) - .merge(::Identity.where(provider: :github, extern_uid: id)) - .first&.id - end - - def format_description(body, author) - return body if cached[:gitlab_user_ids][author.id] - - "*Created by: #{author.username}*\n\n#{body}" - end - - def expire_repository_cache - repository.expire_content_cache if project.repository_exists? - end - - def keep_track_of_errors - return unless errors.any? - - project.update_column(:import_error, { - message: 'The remote data could not be fully imported.', - errors: errors - }.to_json) - end - - def error(type, url, message) - errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } - end - end -end diff --git a/lib/github/import/issue.rb b/lib/github/import/issue.rb deleted file mode 100644 index 171f0872666..00000000000 --- a/lib/github/import/issue.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class Issue < ::Issue - self.table_name = 'issues' - - self.reset_callbacks :save - self.reset_callbacks :create - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/legacy_diff_note.rb b/lib/github/import/legacy_diff_note.rb deleted file mode 100644 index 18adff560b6..00000000000 --- a/lib/github/import/legacy_diff_note.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Github - class Import - class LegacyDiffNote < ::LegacyDiffNote - self.table_name = 'notes' - self.store_full_sti_class = false - - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/merge_request.rb b/lib/github/import/merge_request.rb deleted file mode 100644 index c258e5d5e0e..00000000000 --- a/lib/github/import/merge_request.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class MergeRequest < ::MergeRequest - self.table_name = 'merge_requests' - - self.reset_callbacks :create - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/note.rb b/lib/github/import/note.rb deleted file mode 100644 index 8cf4f30e6b7..00000000000 --- a/lib/github/import/note.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class Note < ::Note - self.table_name = 'notes' - self.store_full_sti_class = false - - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb deleted file mode 100644 index 884693d093c..00000000000 --- a/lib/github/rate_limit.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Github - class RateLimit - SAFE_REMAINING_REQUESTS = 100 - SAFE_RESET_TIME = 500 - RATE_LIMIT_URL = '/rate_limit'.freeze - - attr_reader :connection - - def initialize(connection) - @connection = connection - end - - def get - response = connection.get(RATE_LIMIT_URL) - - # GitHub Rate Limit API returns 404 when the rate limit is disabled - return false unless response.status != 404 - - body = Oj.load(response.body, class_cache: false, mode: :compat) - remaining = body.dig('rate', 'remaining').to_i - reset_in = body.dig('rate', 'reset').to_i - exceed = remaining <= SAFE_REMAINING_REQUESTS - - [exceed, reset_in] - end - end -end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb deleted file mode 100644 index c1c9448f305..00000000000 --- a/lib/github/repositories.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Github - class Repositories - attr_reader :options - - def initialize(options) - @options = options - end - - def fetch - Collection.new(options).fetch(repos_url) - end - - private - - def repos_url - '/user/repos' - end - end -end diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb deleted file mode 100644 index f26bdbdd546..00000000000 --- a/lib/github/representation/base.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Github - module Representation - class Base - def initialize(raw, options = {}) - @raw = raw - @options = options - end - - def id - raw['id'] - end - - def url - raw['url'] - end - - def created_at - raw['created_at'] - end - - def updated_at - raw['updated_at'] - end - - private - - attr_reader :raw, :options - end - end -end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb deleted file mode 100644 index 0087a3d3c4f..00000000000 --- a/lib/github/representation/branch.rb +++ /dev/null @@ -1,55 +0,0 @@ -module Github - module Representation - class Branch < Representation::Base - attr_reader :repository - - def user - raw.dig('user', 'login') || 'unknown' - end - - def repo? - raw['repo'].present? - end - - def repo - return unless repo? - - @repo ||= Github::Representation::Repo.new(raw['repo']) - end - - def ref - raw['ref'] - end - - def sha - raw['sha'] - end - - def short_sha - Commit.truncate_sha(sha) - end - - def valid? - sha.present? && ref.present? - end - - def restore!(name) - repository.create_branch(name, sha) - rescue Gitlab::Git::Repository::InvalidRef => e - Rails.logger.error("#{self.class.name}: Could not restore branch #{name}: #{e}") - end - - def remove!(name) - repository.delete_branch(name) - rescue Gitlab::Git::Repository::DeleteBranchError => e - Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}") - end - - private - - def repository - @repository ||= options.fetch(:repository) - end - end - end -end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb deleted file mode 100644 index 83bf0b5310d..00000000000 --- a/lib/github/representation/comment.rb +++ /dev/null @@ -1,42 +0,0 @@ -module Github - module Representation - class Comment < Representation::Base - def note - raw['body'] || '' - end - - def author - @author ||= Github::Representation::User.new(raw['user'], options) - end - - def commit_id - raw['commit_id'] - end - - def line_code - return unless on_diff? - - parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) - generate_line_code(parsed_lines.to_a.last) - end - - private - - def generate_line_code(line) - Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) - end - - def on_diff? - diff_hunk.present? - end - - def diff_hunk - raw['diff_hunk'] - end - - def file_path - raw['path'] - end - end - end -end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb deleted file mode 100644 index 768ba3b993c..00000000000 --- a/lib/github/representation/issuable.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Github - module Representation - class Issuable < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['body'] || '' - end - - def milestone - return unless raw['milestone'].present? - - @milestone ||= Github::Representation::Milestone.new(raw['milestone']) - end - - def author - @author ||= Github::Representation::User.new(raw['user'], options) - end - - def labels? - raw['labels'].any? - end - - def labels - @labels ||= Array(raw['labels']).map do |label| - Github::Representation::Label.new(label, options) - end - end - end - end -end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb deleted file mode 100644 index 4f1a02cb90f..00000000000 --- a/lib/github/representation/issue.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Github - module Representation - class Issue < Representation::Issuable - def state - raw['state'] == 'closed' ? 'closed' : 'opened' - end - - def comments? - raw['comments'] > 0 - end - - def pull_request? - raw['pull_request'].present? - end - - def assigned? - raw['assignees'].present? - end - - def assignees - @assignees ||= Array(raw['assignees']).map do |user| - Github::Representation::User.new(user, options) - end - end - end - end -end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb deleted file mode 100644 index 60aa51f9569..00000000000 --- a/lib/github/representation/label.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - module Representation - class Label < Representation::Base - def color - "##{raw['color']}" - end - - def title - raw['name'] - end - end - end -end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb deleted file mode 100644 index 917e6394ad4..00000000000 --- a/lib/github/representation/milestone.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Github - module Representation - class Milestone < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['description'] - end - - def due_date - raw['due_on'] - end - - def state - raw['state'] == 'closed' ? 'closed' : 'active' - end - end - end -end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb deleted file mode 100644 index 0171179bb0f..00000000000 --- a/lib/github/representation/pull_request.rb +++ /dev/null @@ -1,71 +0,0 @@ -module Github - module Representation - class PullRequest < Representation::Issuable - delegate :sha, to: :source_branch, prefix: true - delegate :sha, to: :target_branch, prefix: true - - def source_project - project - end - - def source_branch_name - # Mimic the "user:branch" displayed in the MR widget, - # i.e. "Request to merge rymai:add-external-mounts into master" - cross_project? ? "#{source_branch.user}:#{source_branch.ref}" : source_branch.ref - end - - def target_project - project - end - - def target_branch_name - target_branch.ref - end - - def state - return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? - return 'closed' if raw['state'] == 'closed' - - 'opened' - end - - def opened? - state == 'opened' - end - - def valid? - source_branch.valid? && target_branch.valid? - end - - def assigned? - raw['assignee'].present? - end - - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee'], options) - end - - private - - def project - @project ||= options.fetch(:project) - end - - def source_branch - @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) - end - - def target_branch - @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) - end - - def cross_project? - return true unless source_branch.repo? - - source_branch.repo.id != target_branch.repo.id - end - end - end -end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb deleted file mode 100644 index e7e4b428c1a..00000000000 --- a/lib/github/representation/release.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Github - module Representation - class Release < Representation::Base - def description - raw['body'] - end - - def tag - raw['tag_name'] - end - - def valid? - !raw['draft'] - end - end - end -end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb deleted file mode 100644 index 6938aa7db05..00000000000 --- a/lib/github/representation/repo.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Github - module Representation - class Repo < Representation::Base - end - end -end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb deleted file mode 100644 index 18591380e25..00000000000 --- a/lib/github/representation/user.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Github - module Representation - class User < Representation::Base - def email - return @email if defined?(@email) - - @email = Github::User.new(username, options).get.fetch('email', nil) - end - - def username - raw['login'] - end - end - end -end diff --git a/lib/github/response.rb b/lib/github/response.rb deleted file mode 100644 index 761c524b553..00000000000 --- a/lib/github/response.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Github - class Response - attr_reader :raw, :headers, :status - - def initialize(response) - @raw = response - @headers = response.headers - @status = response.status - end - - def body - Oj.load(raw.body, class_cache: false, mode: :compat) - end - - def rels - links = headers['Link'].to_s.split(', ').map do |link| - href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures - - [name.to_sym, href] - end - - Hash[*links.flatten] - end - end -end diff --git a/lib/github/user.rb b/lib/github/user.rb deleted file mode 100644 index f88a29e590b..00000000000 --- a/lib/github/user.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Github - class User - attr_reader :username, :options - - def initialize(username, options) - @username = username - @options = options - end - - def get - client.get(user_url).body - end - - private - - def client - @client ||= Github::Client.new(options) - end - - def user_url - "/users/#{username}" - end - end -end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index c730fefcffe..eeb03625479 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -8,7 +8,7 @@ module Gitlab ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ - ImportSource.new('github', 'GitHub', Github::Import), + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 7f86fd7b45e..8e9aef49e89 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -7,7 +7,7 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @options = { token: token, verbose: true } + @options = { token: token } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first @@ -42,7 +42,9 @@ class GithubImport import_success = false timings = Benchmark.measure do - import_success = Github::Import.new(@project, @options).execute + import_success = Gitlab::GithubImport::SequentialImporter + .new(@project, token: @options[:token]) + .execute end if import_success -- cgit v1.2.1 From fec48c6e170fb0032cece5d8cc3b06bb45caee57 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 7 Nov 2017 17:11:37 +0100 Subject: Use Commit#notes and Note.for_commit_id when possible to make sure we use all the indexes available to us --- lib/api/commits.rb | 2 +- lib/api/v3/commits.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 2685dc27252..2bc4039b019 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -117,7 +117,7 @@ module API commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - notes = user_project.notes.where(commit_id: commit.id).order(:created_at) + notes = commit.notes.order(:created_at) present paginate(notes), with: Entities::CommitNote end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index ed206a6def0..be360fbfc0c 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -106,7 +106,7 @@ module API commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - notes = Note.where(commit_id: commit.id).order(:created_at) + notes = commit.notes.order(:created_at) present paginate(notes), with: ::API::Entities::CommitNote end -- cgit v1.2.1 From 00ce8756ad18987eee622c9aaaa888bc0207ed1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Thu, 2 Nov 2017 15:08:39 +0100 Subject: Migrate GitLab::Git::Wiki.update_page to Gitaly --- lib/gitlab/git/wiki.rb | 25 ++++++++++++++++++++----- lib/gitlab/gitaly_client/wiki_service.rb | 25 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index fe901d049d4..5d36f16abd4 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -48,11 +48,14 @@ module Gitlab end def update_page(page_path, title, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) - nil + @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| + if is_enabled + gitaly_update_page(page_path, title, format, content, commit_details) + gollum_wiki.clear_cache + else + gollum_update_page(page_path, title, format, content, commit_details) + end + end end def pages @@ -149,6 +152,14 @@ module Gitlab nil end + def gollum_update_page(page_path, title, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + nil + end + def gollum_find_page(title:, version: nil, dir: nil) if version version = Gitlab::Git::Commit.find(@repository, version).id @@ -172,6 +183,10 @@ module Gitlab gitaly_wiki_client.write_page(name, format, content, commit_details) end + def gitaly_update_page(page_path, title, format, content, commit_details) + gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) + end + def gitaly_delete_page(page_path, commit_details) gitaly_wiki_client.delete_page(page_path, commit_details) end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 15f0f30d303..1a668338f57 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -37,6 +37,31 @@ module Gitlab end end + def update_page(page_path, title, format, content, commit_details) + request = Gitaly::WikiUpdatePageRequest.new( + repository: @gitaly_repo, + page_path: GitalyClient.encode(page_path), + title: GitalyClient.encode(title), + format: format.to_s, + commit_details: gitaly_commit_details(commit_details) + ) + + strio = StringIO.new(content) + + enum = Enumerator.new do |y| + until strio.eof? + chunk = strio.read(MAX_MSG_SIZE) + request.content = GitalyClient.encode(chunk) + + y.yield request + + request = Gitaly::WikiUpdatePageRequest.new + end + end + + GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum) + end + def delete_page(page_path, commit_details) request = Gitaly::WikiDeletePageRequest.new( repository: @gitaly_repo, -- cgit v1.2.1 From 0232450c8aee08a656275caf7b990e0bcbbb1cf4 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 8 Nov 2017 16:21:39 +0000 Subject: Fix Error 500 when pushing LFS objects with a write deploy key --- lib/gitlab/auth.rb | 15 +++++++++++---- lib/gitlab/lfs_token.rb | 4 ++++ 2 files changed, 15 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 0ad9285c0ea..cbbc51db99e 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -25,7 +25,7 @@ module Gitlab result = service_request_check(login, password, project) || build_access_token_check(login, password) || - lfs_token_check(login, password) || + lfs_token_check(login, password, project) || oauth_access_token_check(login, password) || personal_access_token_check(password) || user_with_password_for_git(login, password) || @@ -146,7 +146,7 @@ module Gitlab end.flatten.uniq end - def lfs_token_check(login, password) + def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) actor = @@ -163,6 +163,8 @@ module Gitlab authentication_abilities = if token_handler.user? full_authentication_abilities + elsif token_handler.deploy_key_pushable?(project) + read_write_authentication_abilities else read_authentication_abilities end @@ -208,10 +210,15 @@ module Gitlab ] end - def full_authentication_abilities + def read_write_authentication_abilities read_authentication_abilities + [ :push_code, - :create_container_image, + :create_container_image + ] + end + + def full_authentication_abilities + read_write_authentication_abilities + [ :admin_container_image ] end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 8e57ba831c5..ead5d566871 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -27,6 +27,10 @@ module Gitlab end end + def deploy_key_pushable?(project) + actor.is_a?(DeployKey) && actor.can_push_to?(project) + end + def user? actor.is_a?(User) end -- cgit v1.2.1 From 2b886a7815fc6e717e6aabc92380679e1dcaf0ee Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 8 Nov 2017 18:06:03 +0100 Subject: Restore Enterprise support in the GH importer This was removed by accident as the old GitHub importer handled this deep down the codebase, making it easy to miss. --- lib/gitlab/github_import/client.rb | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 844530b1ea7..c1c338487a7 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -38,7 +38,14 @@ module Gitlab # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. def initialize(token, per_page: 100, parallel: true) - @octokit = Octokit::Client.new(access_token: token, per_page: per_page) + @octokit = Octokit::Client.new( + access_token: token, + per_page: per_page, + api_endpoint: api_endpoint + ) + + @octokit.connection_options[:ssl] = { verify: verify_ssl } + @parallel = parallel end @@ -163,8 +170,27 @@ module Gitlab octokit.rate_limit.resets_in + 5 end - def respond_to_missing?(method, include_private = false) - octokit.respond_to?(method, include_private) + def api_endpoint + custom_api_endpoint || default_api_endpoint + end + + def custom_api_endpoint + github_omniauth_provider.dig('args', 'client_options', 'site') + end + + def default_api_endpoint + OmniAuth::Strategies::GitHub.default_options[:client_options][:site] + end + + def verify_ssl + github_omniauth_provider.fetch('verify_ssl', true) + end + + def github_omniauth_provider + @github_omniauth_provider ||= + Gitlab.config.omniauth.providers + .find { |provider| provider.name == 'github' } + .to_h end def rate_limit_counter -- cgit v1.2.1 From ff082a003d103736cef8a4d58213f8366f45bdcd Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 8 Nov 2017 18:17:27 +0100 Subject: Fix the GH importer Rake task This task was broken in a few areas with the removal of the old GitHub importer code. --- lib/tasks/import.rake | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 8e9aef49e89..943cbe6d80c 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -14,7 +14,9 @@ class GithubImport end def run! - @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one! + @repo = GithubRepos + .new(@options[:token], @current_user, @github_repo) + .choose_one! raise 'No repo found!' unless @repo @@ -28,7 +30,7 @@ class GithubImport private def show_warning! - puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" + puts "This will import GitHub #{@repo.full_name.bright} into GitLab #{@project_path.bright} as #{@current_user.name}" puts "Permission checks are ignored. Press any key to continue.".color(:red) STDIN.getch @@ -65,16 +67,16 @@ class GithubImport @current_user, name: name, path: name, - description: @repo['description'], + description: @repo.description, namespace_id: namespace.id, visibility_level: visibility_level, - skip_wiki: @repo['has_wiki'] + skip_wiki: @repo.has_wiki ).execute project.update!( import_type: 'github', - import_source: @repo['full_name'], - import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@") + import_source: @repo.full_name, + import_url: @repo.clone_url.sub('://', "://#{@options[:token]}@") ) project @@ -93,13 +95,15 @@ class GithubImport end def visibility_level - @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility + @repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility end end class GithubRepos - def initialize(options, current_user, github_repo) - @options = options + def initialize(token, current_user, github_repo) + @client = Octokit::Client + .new(access_token: token, auto_paginate: true, per_page: 100) + @current_user = current_user @github_repo = github_repo end @@ -108,17 +112,17 @@ class GithubRepos return found_github_repo if @github_repo repos.each do |repo| - print "ID: #{repo['id'].to_s.bright}".color(:green) - print "\tName: #{repo['full_name']}\n".color(:green) + print "ID: #{repo.id.to_s.bright}".color(:green) + print "\tName: #{repo.full_name}\n".color(:green) end print 'ID? '.bright - repos.find { |repo| repo['id'] == repo_id } + repos.find { |repo| repo.id == repo_id } end def found_github_repo - repos.find { |repo| repo['full_name'] == @github_repo } + repos.find { |repo| repo.full_name == @github_repo } end def repo_id @@ -126,7 +130,7 @@ class GithubRepos end def repos - Github::Repositories.new(@options).fetch + @client.list_repositories end end -- cgit v1.2.1 From 12d622eb996b6499e5fbd2be01cca27c08a976fa Mon Sep 17 00:00:00 2001 From: Mark Fletcher Date: Wed, 8 Nov 2017 10:22:24 +0000 Subject: Fix acceptance of username for Mattermost service update via API --- lib/api/services.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/api/services.rb b/lib/api/services.rb index 6454e475036..bbcc851d07a 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -522,6 +522,12 @@ module API name: :webhook, type: String, desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + }, + { + required: false, + name: :username, + type: String, + desc: 'The username to use to post the message' } ], 'teamcity' => [ -- cgit v1.2.1 From 48cb1c509686fc40f65ad153f36ab5d22feb63ac Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 8 Nov 2017 21:20:46 +0100 Subject: Restore GH enterprise support in the Rake task This restores GH enterprise support in the GH import Rake task. --- lib/tasks/import.rake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 943cbe6d80c..aafbe52e5f8 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -101,8 +101,8 @@ end class GithubRepos def initialize(token, current_user, github_repo) - @client = Octokit::Client - .new(access_token: token, auto_paginate: true, per_page: 100) + @client = Gitlab::GithubImport::Client.new(token) + @client.octokit.auto_paginate = true @current_user = current_user @github_repo = github_repo @@ -130,7 +130,7 @@ class GithubRepos end def repos - @client.list_repositories + @client.octokit.list_repositories end end -- cgit v1.2.1 From f37fe2edc80b83513cb9d30b6c97e85c9ccca29c Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 8 Nov 2017 21:37:01 +0100 Subject: Support importing GH projects without rate limits GitHub Enterprise disables rate limiting for the API, resulting in HTTP 404 errors when requesting rate limiting details. This changes Gitlab::GithubImport::Client so it can deal with rate limiting being disabled. --- lib/gitlab/github_import/client.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index c1c338487a7..5da9befa08e 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -129,6 +129,8 @@ module Gitlab # whether we are running in parallel mode or not. For more information see # `#rate_or_wait_for_rate_limit`. def with_rate_limit + return yield unless rate_limiting_enabled? + request_count_counter.increment raise_or_wait_for_rate_limit unless requests_remaining? @@ -170,6 +172,10 @@ module Gitlab octokit.rate_limit.resets_in + 5 end + def rate_limiting_enabled? + @rate_limiting_enabled ||= api_endpoint.include?('.github.com') + end + def api_endpoint custom_api_endpoint || default_api_endpoint end -- cgit v1.2.1 From 7fd3ce417f50c6a5f4f4d852514e5f4f03945327 Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Wed, 8 Nov 2017 16:21:08 -0500 Subject: Revert "add metrics tagging to the sidekiq middleware" This reverts commit d5859bb9d59b3750ac6e9b8c4c17d69c4c3ed077. This reverts commit 2b7e03cf699f9d266af585a1a9399c3e219fe063. This reverts commit 7799a9bc442738935104d3b047c257e5c5884d95. --- lib/gitlab/metrics/sidekiq_middleware.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index 55c707d5386..df4bdf16847 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -11,8 +11,6 @@ module Gitlab # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } - - worker.metrics_tags.each { |tag, value| trans.add_tag(tag, value) } if worker.respond_to?(:metrics_tags) rescue Exception => error # rubocop: disable Lint/RescueException trans.add_event(:sidekiq_exception) -- cgit v1.2.1 From 89bd78352e4c575a0293f9c431dd677d288d28d2 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 7 Nov 2017 08:30:38 +0000 Subject: Merge branch 'ssrf-protections-round-2' into 'security-10-1' Replace SSRF resolver with Addrinfo.getaddrinfo to include alternative localhost versions See merge request gitlab/gitlabhq!2219 (cherry picked from commit 4a1e73783d5480aa514db7b53e10c075f95580b5) 1bffa0c3 Replace SSRF resolver with Addrinfo.getaddrinfo to include alternative localhost versions --- lib/gitlab/url_blocker.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index fee1a127fd7..13150ddab67 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -22,10 +22,12 @@ module Gitlab return true if blocked_user_or_hostname?(uri.user) return true if blocked_user_or_hostname?(uri.hostname) - server_ips = Resolv.getaddresses(uri.hostname) + server_ips = Addrinfo.getaddrinfo(uri.hostname, 80, nil, :STREAM).map(&:ip_address) return true if (blocked_ips & server_ips).any? rescue Addressable::URI::InvalidURIError return true + rescue SocketError + return false end false -- cgit v1.2.1 From 20ac30a705f4edd22efd934ecf68b58557f868db Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 1 Nov 2017 09:25:49 +0000 Subject: Merge branch '36099-api-responses-missing-x-content-type-options-header' into '10-1-stable' Include X-Content-Type-Options (XCTO) header into API responses See merge request gitlab/gitlabhq!2211 (cherry picked from commit 6c818e77f2abeef2dd7b17a269611b018701fa79) e087e075 Include X-Content-Type-Options (XCTO) header into API responses --- lib/api/api.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index c37e596eb9d..8094597d238 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -61,7 +61,10 @@ module API mount ::API::V3::Variables end - before { header['X-Frame-Options'] = 'SAMEORIGIN' } + before do + header['X-Frame-Options'] = 'SAMEORIGIN' + header['X-Content-Type-Options'] = 'nosniff' + end # The locale is set to the current user's locale when `current_user` is loaded after { Gitlab::I18n.use_default_locale } -- cgit v1.2.1 From 90d74ce2dfbb189c29ad01b8eff20ad0b9713b2c Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 9 Nov 2017 09:32:21 +0100 Subject: Repository Exists check is OPT_OUT for Gitaly Moving more git operations to be executed by Gitaly, now the check if a repository exists is an opt out endpoint. Can be disabled, for the time being, by performing in the rails console: > Feature.get('gitaly_repository_exists').disable => true Part of gitlab-org/gitaly#314 --- lib/gitlab/git/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d236e1b03e6..359547cf63d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -105,7 +105,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_repository_client.exists? else -- cgit v1.2.1 From 25eea058335ed0149621fb7470c9e8db5a24b481 Mon Sep 17 00:00:00 2001 From: Tim Zallmann Date: Thu, 9 Nov 2017 10:38:00 +0100 Subject: Icon Sprite URL is also local even if asset_host is set --- lib/gitlab/gon_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3a666c2268b..dfcdfc307b6 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -20,7 +20,7 @@ module Gitlab gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') - gon.sprite_icons = ActionController::Base.helpers.asset_path('icons.svg') + gon.sprite_icons = IconsHelper.sprite_icon_path if current_user gon.current_user_id = current_user.id -- cgit v1.2.1 From ebd51744729cb1b68754f8ba4d7f9adcec28d58d Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Wed, 8 Nov 2017 12:59:48 +0000 Subject: Handle forks in Gitlab::Checks::LfsIntegrity --- lib/gitlab/checks/lfs_integrity.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index 27a95764dc1..f7276a380dc 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -15,7 +15,10 @@ module Gitlab return false unless new_lfs_pointers.present? - existing_count = @project.lfs_objects.where(oid: new_lfs_pointers.map(&:lfs_oid)).count + existing_count = @project.lfs_storage_project + .lfs_objects + .where(oid: new_lfs_pointers.map(&:lfs_oid)) + .count existing_count != new_lfs_pointers.count end -- cgit v1.2.1 From 26b6dfaf6f74a8b1e9cc1b05c30879f8a60f86bb Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 9 Nov 2017 16:07:04 +0000 Subject: Add /groups/:id/subgroups endpoint to API --- lib/api/groups.rb | 66 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 340a7cecf09..bcf2e6dae1d 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -25,24 +25,7 @@ module API optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end - def present_groups(groups, options = {}) - options = options.reverse_merge( - with: Entities::Group, - current_user: current_user - ) - - groups = groups.with_statistics if options[:statistics] - present paginate(groups), options - end - end - - resource :groups do - include CustomAttributesEndpoints - - desc 'Get a groups list' do - success Entities::Group - end - params do + params :group_list_params do use :statistics_params optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' optional :all_available, type: Boolean, desc: 'Show all group that you have access to' @@ -52,19 +35,47 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end - get do + + def find_groups(params) find_params = { all_available: params[:all_available], - owned: params[:owned], - custom_attributes: params[:custom_attributes] + custom_attributes: params[:custom_attributes], + owned: params[:owned] } + find_params[:parent] = find_group!(params[:id]) if params[:id] groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present_groups groups, statistics: params[:statistics] && current_user.admin? + groups + end + + def present_groups(params, groups) + options = { + with: Entities::Group, + current_user: current_user, + statistics: params[:statistics] && current_user.admin? + } + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end + end + + resource :groups do + include CustomAttributesEndpoints + + desc 'Get a groups list' do + success Entities::Group + end + params do + use :group_list_params + end + get do + groups = find_groups(params) + present_groups params, groups end desc 'Create a group. Available only for users who can create groups.' do @@ -166,6 +177,17 @@ module API present paginate(projects), with: entity, current_user: current_user end + desc 'Get a list of subgroups in this group.' do + success Entities::Group + end + params do + use :group_list_params + end + get ":id/subgroups" do + groups = find_groups(params) + present_groups params, groups + end + desc 'Transfer a project to the group namespace. Available only for admin.' do success Entities::GroupDetail end -- cgit v1.2.1 From d825c9cb5d89d18685a196789b477df83998fed2 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 6 Nov 2017 14:46:27 +0100 Subject: Clean up schema of the "issues" table This adds various foreign key constraints, indexes, missing NOT NULL constraints, and changes some column types from timestamp to timestamptz. Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/31811 --- lib/gitlab/hook_data/issue_builder.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index de9cab80a02..196f2b6b34c 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -4,7 +4,6 @@ module Gitlab SAFE_HOOK_ATTRIBUTES = %i[ assignee_id author_id - branch_name closed_at confidential created_at -- cgit v1.2.1 From de301d13bb4f6d61cdaebfe186be6012c2f2096d Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Fri, 10 Nov 2017 14:45:23 +0000 Subject: Prepare Repository#fetch_source_branch for migration --- lib/gitlab/git/operation_service.rb | 10 ++--- lib/gitlab/git/remote_repository.rb | 82 +++++++++++++++++++++++++++++++++++++ lib/gitlab/git/repository.rb | 34 +++++---------- 3 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 lib/gitlab/git/remote_repository.rb (limited to 'lib') diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ab94ba8a73a..e36d5410431 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -72,7 +72,7 @@ module Gitlab # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, # it would be created from `start_branch_name`. - # If `start_project` is passed, and the branch doesn't exist, + # If `start_repository` is passed, and the branch doesn't exist, # it would try to find the commits from it instead of current repository. def with_branch( branch_name, @@ -80,15 +80,13 @@ module Gitlab start_repository: repository, &block) - # Refactoring aid - unless start_repository.is_a?(Gitlab::Git::Repository) - raise "expected a Gitlab::Git::Repository, got #{start_repository}" - end + Gitlab::Git.check_namespace!(start_repository) + start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) start_branch_name = nil if start_repository.empty_repo? if start_branch_name && !start_repository.branch_exists?(start_branch_name) - raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.full_path}" + raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" end update_branch_with_hooks(branch_name) do diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb new file mode 100644 index 00000000000..3685aa20669 --- /dev/null +++ b/lib/gitlab/git/remote_repository.rb @@ -0,0 +1,82 @@ +module Gitlab + module Git + # + # When a Gitaly call involves two repositories instead of one we cannot + # assume that both repositories are on the same Gitaly server. In this + # case we need to make a distinction between the repository that the + # call is being made on (a Repository instance), and the "other" + # repository (a RemoteRepository instance). This is the reason why we + # have the RemoteRepository class in Gitlab::Git. + # + # When you make changes, be aware that gitaly-ruby sub-classes this + # class. + # + class RemoteRepository + attr_reader :path, :relative_path, :gitaly_repository + + def initialize(repository) + @relative_path = repository.relative_path + @gitaly_repository = repository.gitaly_repository + + # These instance variables will not be available in gitaly-ruby, where + # we have no disk access to this repository. + @repository = repository + @path = repository.path + end + + def empty_repo? + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + @repository.empty_repo? + end + + def commit_id(revision) + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + @repository.commit(revision)&.sha + end + + def branch_exists?(name) + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + @repository.branch_exists?(name) + end + + # Compares self to a Gitlab::Git::Repository. This implementation uses + # 'self.gitaly_repository' so that it will also work in the + # GitalyRemoteRepository subclass defined in gitaly-ruby. + def same_repository?(other_repository) + gitaly_repository.storage_name == other_repository.storage && + gitaly_repository.relative_path == other_repository.relative_path + end + + def fetch_env + gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) + gitaly_address = gitaly_client.address(storage) + gitaly_token = gitaly_client.token(storage) + + request = Gitaly::SSHUploadPackRequest.new(repository: gitaly_repository) + env = { + 'GITALY_ADDRESS' => gitaly_address, + 'GITALY_PAYLOAD' => request.to_json, + 'GITALY_WD' => Dir.pwd, + 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" + } + env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? + + env + end + + private + + # Must return an object that responds to 'address' and 'storage'. + def gitaly_client + Gitlab::GitalyClient + end + + def storage + gitaly_repository.storage_name + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d236e1b03e6..14a8d551524 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -58,7 +58,7 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver + attr_reader :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -66,7 +66,6 @@ module Gitlab @storage = storage @relative_path = relative_path @gl_repository = gl_repository - @gitaly_resolver = Gitlab::GitalyClient storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -1014,23 +1013,22 @@ module Gitlab def with_repo_branch_commit(start_repository, start_branch_name) Gitlab::Git.check_namespace!(start_repository) + start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) return yield nil if start_repository.empty_repo? - if start_repository == self + if start_repository.same_repository?(self) yield commit(start_branch_name) else - start_commit = start_repository.commit(start_branch_name) + start_commit_id = start_repository.commit_id(start_branch_name) - return yield nil unless start_commit + return yield nil unless start_commit_id - sha = start_commit.sha - - if branch_commit = commit(sha) + if branch_commit = commit(start_commit_id) yield branch_commit else with_repo_tmp_commit( - start_repository, start_branch_name, sha) do |tmp_commit| + start_repository, start_branch_name, start_commit_id) do |tmp_commit| yield tmp_commit end end @@ -1087,6 +1085,9 @@ module Gitlab end def fetch_ref(source_repository, source_ref:, target_ref:) + Gitlab::Git.check_namespace!(source_repository) + source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) + message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| if is_enabled gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) @@ -1620,22 +1621,9 @@ module Gitlab end def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) - gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) - gitaly_address = gitaly_resolver.address(source_repository.storage) - gitaly_token = gitaly_resolver.token(source_repository.storage) - - request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository) - env = { - 'GITALY_ADDRESS' => gitaly_address, - 'GITALY_PAYLOAD' => request.to_json, - 'GITALY_WD' => Dir.pwd, - 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" - } - env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? - args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) - run_git(args, env: env) + run_git(args, env: source_repository.fetch_env) end def gitaly_ff_merge(user, source_sha, target_branch) -- cgit v1.2.1 From 258bf3e187538bd326491e5d1b25a0511fbd96a1 Mon Sep 17 00:00:00 2001 From: "Lin Jen-Shin (godfat)" Date: Mon, 13 Nov 2017 15:27:30 +0000 Subject: Add Gitlab::Utils::StrongMemoize --- lib/api/api_guard.rb | 8 +++++--- lib/gitlab/utils/strong_memoize.rb | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 lib/gitlab/utils/strong_memoize.rb (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index b9c7d443f6c..c1c0d344917 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -42,6 +42,8 @@ module API # Helper Methods for Grape Endpoint module HelperMethods + include Gitlab::Utils::StrongMemoize + def find_current_user! user = find_user_from_access_token || find_user_from_warden return unless user @@ -52,9 +54,9 @@ module API end def access_token - return @access_token if defined?(@access_token) - - @access_token = find_oauth_access_token || find_personal_access_token + strong_memoize(:access_token) do + find_oauth_access_token || find_personal_access_token + end end def validate_access_token!(scopes: []) diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb new file mode 100644 index 00000000000..a2ac9285b56 --- /dev/null +++ b/lib/gitlab/utils/strong_memoize.rb @@ -0,0 +1,31 @@ +module Gitlab + module Utils + module StrongMemoize + # Instead of writing patterns like this: + # + # def trigger_from_token + # return @trigger if defined?(@trigger) + # + # @trigger = Ci::Trigger.find_by_token(params[:token].to_s) + # end + # + # We could write it like: + # + # def trigger_from_token + # strong_memoize(:trigger) do + # Ci::Trigger.find_by_token(params[:token].to_s) + # end + # end + # + def strong_memoize(name) + ivar_name = "@#{name}" + + if instance_variable_defined?(ivar_name) + instance_variable_get(ivar_name) + else + instance_variable_set(ivar_name, yield) + end + end + end + end +end -- cgit v1.2.1 From 1162d89ac49553c579ec4d049e74206893ff6302 Mon Sep 17 00:00:00 2001 From: Travis Miller Date: Mon, 13 Nov 2017 16:05:44 +0000 Subject: Add administrative endpoint to list all pages domains --- lib/api/entities.rb | 20 ++++++++++++++++++-- lib/api/helpers.rb | 9 +++++++++ lib/api/pages_domains.rb | 22 +++++++++++++++++++++- 3 files changed, 48 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index a382db92e8d..16ae99b5c6c 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1042,6 +1042,11 @@ module API expose :value end + class PagesDomainCertificateExpiration < Grape::Entity + expose :expired?, as: :expired + expose :expiration + end + class PagesDomainCertificate < Grape::Entity expose :subject expose :expired?, as: :expired @@ -1049,12 +1054,23 @@ module API expose :certificate_text end + class PagesDomainBasic < Grape::Entity + expose :domain + expose :url + expose :certificate, + as: :certificate_expiration, + if: ->(pages_domain, _) { pages_domain.certificate? }, + using: PagesDomainCertificateExpiration do |pages_domain| + pages_domain + end + end + class PagesDomain < Grape::Entity expose :domain expose :url expose :certificate, - if: ->(pages_domain, _) { pages_domain.certificate? }, - using: PagesDomainCertificate do |pages_domain| + if: ->(pages_domain, _) { pages_domain.certificate? }, + using: PagesDomainCertificate do |pages_domain| pages_domain end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 5f9b94cc89c..3c8960cb1ab 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -155,6 +155,11 @@ module API end end + def authenticated_with_full_private_access! + authenticate! + forbidden! unless current_user.full_private_access? + end + def authenticated_as_admin! authenticate! forbidden! unless current_user.admin? @@ -190,6 +195,10 @@ module API not_found! unless user_project.pages_available? end + def require_pages_config_enabled! + not_found! unless Gitlab.config.pages.enabled + end + def can?(object, action, subject = :global) Ability.allowed?(object, action, subject) end diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 259f3f34068..d7b613a717e 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -4,7 +4,6 @@ module API before do authenticate! - require_pages_enabled! end after_validation do @@ -29,10 +28,31 @@ module API end end + resource :pages do + before do + require_pages_config_enabled! + authenticated_with_full_private_access! + end + + desc "Get all pages domains" do + success Entities::PagesDomainBasic + end + params do + use :pagination + end + get "domains" do + present paginate(PagesDomain.all), with: Entities::PagesDomainBasic + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: { id: %r{[^/]+} } do + before do + require_pages_enabled! + end + desc 'Get all pages domains' do success Entities::PagesDomain end -- cgit v1.2.1 From 282e7f8eab2b0f2579d919fd7e1e4a50dc5505e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Thu, 9 Nov 2017 20:27:00 -0300 Subject: Incorporate Gitaly's WikiService.WikiGetAllPages RPC --- lib/gitlab/git/wiki.rb | 18 +++++++++- lib/gitlab/gitaly_client/wiki_page.rb | 4 +++ lib/gitlab/gitaly_client/wiki_service.rb | 58 ++++++++++++++++++++++---------- 3 files changed, 62 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 5d36f16abd4..022d1f249a9 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -59,7 +59,13 @@ module Gitlab end def pages - gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| + if is_enabled + gitaly_get_all_pages + else + gollum_get_all_pages + end + end end def page(title:, version: nil, dir: nil) @@ -179,6 +185,10 @@ module Gitlab Gitlab::Git::WikiFile.new(gollum_file) end + def gollum_get_all_pages + gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + end + def gitaly_write_page(name, format, content, commit_details) gitaly_wiki_client.write_page(name, format, content, commit_details) end @@ -204,6 +214,12 @@ module Gitlab Gitlab::Git::WikiFile.new(wiki_file) end + + def gitaly_get_all_pages + gitaly_wiki_client.get_all_pages.map do |wiki_page, version| + Gitlab::Git::WikiPage.new(wiki_page, version) + end + end end end end diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index 8226278d5f6..98d96fe6211 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -11,6 +11,10 @@ module Gitlab FIELDS.each do |field| instance_variable_set("@#{field}", params[field]) end + + # All gRPC strings in a response are frozen, so we get an unfrozen + # version here so appending to `raw_data` doesn't blow up. + @raw_data = @raw_data.dup end def historical? diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 1a668338f57..8f05f40365e 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -81,28 +81,23 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) - wiki_page = version = nil - response.each do |message| - page = message.page - next unless page + wiki_page_from_iterator(response) + end - if wiki_page - wiki_page.raw_data << page.raw_data - else - wiki_page = GitalyClient::WikiPage.new(page.to_h) - # All gRPC strings in a response are frozen, so we get - # an unfrozen version here so appending in the else clause below doesn't blow up. - wiki_page.raw_data = wiki_page.raw_data.dup + def get_all_pages + request = Gitaly::WikiGetAllPagesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request) + pages = [] - version = Gitlab::Git::WikiPageVersion.new( - Gitlab::Git::Commit.decorate(@repository, page.version.commit), - page.version.format - ) - end + loop do + page, version = wiki_page_from_iterator(response) { |message| message.end_of_page } + + break unless page && version + pages << [page, version] end - [wiki_page, version] + pages end def find_file(name, revision) @@ -133,6 +128,35 @@ module Gitlab private + # If a block is given and the yielded value is true, iteration will be + # stopped early at that point; else the iterator is consumed entirely. + # The iterator is traversed with `next` to allow resuming the iteration. + def wiki_page_from_iterator(iterator) + wiki_page = version = nil + + while message = iterator.next + break if block_given? && yield(message) + + page = message.page + next unless page + + if wiki_page + wiki_page.raw_data << page.raw_data + else + wiki_page = GitalyClient::WikiPage.new(page.to_h) + + version = Gitlab::Git::WikiPageVersion.new( + Gitlab::Git::Commit.decorate(@repository, page.version.commit), + page.version.format + ) + end + end + + [wiki_page, version] + rescue StopIteration + [wiki_page, version] + end + def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( name: GitalyClient.encode(commit_details.name), -- cgit v1.2.1 From aaf18bb8c8238891458074e0712f5963005b8436 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 14 Nov 2017 10:33:20 +0100 Subject: Don't try to create fork network memberships for forks of forks In case the root project of a Fork-of-fork is deleted, the ForkNetwork and the membership for that fork network is never created. In this case we shouldn't try to create the membership, since the parent membership will never be created. This means that these fork networks will be lost. --- .../create_fork_network_memberships_range.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index c88eb9783ed..67a39d28944 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -51,10 +51,20 @@ module Gitlab FROM projects WHERE forked_project_links.forked_from_project_id = projects.id ) + AND NOT EXISTS ( + SELECT true + FROM forked_project_links AS parent_links + WHERE parent_links.forked_to_project_id = forked_project_links.forked_from_project_id + AND NOT EXISTS ( + SELECT true + FROM projects + WHERE parent_links.forked_from_project_id = projects.id + ) + ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} MISSING_MEMBERS - ForkNetworkMember.count_by_sql(count_sql) > 0 + ForkedProjectLink.count_by_sql(count_sql) > 0 end def log(message) -- cgit v1.2.1 From 52bfd064627c6a63b82bc81e7ebabac299b25eed Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Tue, 14 Nov 2017 16:14:35 +0100 Subject: Use relative git object paths to construct absolute ones before setting Env --- lib/api/helpers/internal_helpers.rb | 12 ++++++++++++ lib/api/internal.rb | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 4c0db4d42b1..4b3c473b0bb 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -36,6 +36,18 @@ module API {} end + def fix_git_env_repository_paths(env, repository_path) + if obj_dir_relative = env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence + env['GIT_OBJECT_DIRECTORY'] = File.join(repository_path, obj_dir_relative) + end + + if alt_obj_dirs_relative = env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE'].presence + env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_obj_dirs_relative.map { |dir| File.join(repository_path, dir) } + end + + env + end + def log_user_activity(actor) commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 6e78ac2c903..451121a4cea 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -19,7 +19,9 @@ module API status 200 # Stores some Git-specific env thread-safely - Gitlab::Git::Env.set(parse_env) + env = parse_env + env = fix_git_env_repository_paths(env, repository_path) if project + Gitlab::Git::Env.set(env) actor = if params[:key_id] -- cgit v1.2.1 From b5b35865435ee12be26c22d37b286a4936b701af Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 14 Nov 2017 15:32:45 -0600 Subject: Regenerate emoji digests with latest gemojione --- lib/tasks/gemojione.rake | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 87ca39b079b..7a814da36bc 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -1,5 +1,28 @@ namespace :gemojione do desc 'Generates Emoji SHA256 digests' + + task aliases: ['yarn:check', 'environment'] do + require 'json' + + aliases = {} + + index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json') + index = JSON.parse(File.read(index_file)) + + index.each_pair do |key, data| + data['aliases'].each do |a| + a.tr!(':', '') + + aliases[a] = key + end + end + + out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') + File.open(out, 'w') do |handle| + handle.write(JSON.pretty_generate(aliases, indent: ' ', space: '', space_before: '')) + end + end + task digests: ['yarn:check', 'environment'] do require 'digest/sha2' require 'json' @@ -29,7 +52,6 @@ namespace :gemojione do end out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') - File.open(out, 'w') do |handle| handle.write(JSON.pretty_generate(resultant_emoji_map)) end -- cgit v1.2.1 From 8da236611b5182a1105111904027ae3e74ed1682 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 15 Nov 2017 13:27:37 +0100 Subject: Prefer polymorphism over specific type checks in Import service --- lib/gitlab/github_import/parallel_importer.rb | 4 ++++ lib/gitlab/import_export/importer.rb | 4 ++++ 2 files changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 81739834b41..6da11e6ef08 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -11,6 +11,10 @@ module Gitlab true end + def self.imports_repository? + true + end + def initialize(project) @project = project end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index fbdd74788bc..c14646b0611 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,6 +1,10 @@ module Gitlab module ImportExport class Importer + def self.imports_repository? + true + end + def initialize(project) @archive_file = project.import_source @current_user = project.creator -- cgit v1.2.1 From ce1a1aa1f67d0438fd534920c9aef4e068ea4ddd Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 15 Nov 2017 07:36:19 -0600 Subject: Move :gay_pride_flag: to flags category --- lib/tasks/gemojione.rake | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 7a814da36bc..c2d3a6b6950 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -39,8 +39,13 @@ namespace :gemojione do fpath = File.join(dir, "#{emoji_hash['unicode']}.png") hash_digest = Digest::SHA256.file(fpath).hexdigest + category = emoji_hash['category'] + if name == 'gay_pride_flag' + category = 'flags' + end + entry = { - category: emoji_hash['category'], + category: category, moji: emoji_hash['moji'], description: emoji_hash['description'], unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), -- cgit v1.2.1 From 54533920bc81319159297c7f7c1c8c2451baf151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 15 Nov 2017 15:18:07 +0100 Subject: Always fetch master from the canonical remote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This way, if a fork's master branch is far behind the canonical's master, it won't use the tip of the fork's master branch as merge base and thus won't create a huge and wrong patch file. Signed-off-by: Rémy Coutable --- lib/gitlab/ee_compat_check.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 0ea534a5fd0..efc2e46d289 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -193,7 +193,7 @@ module Gitlab # Repository is initially cloned with a depth of 20 so we need to fetch # deeper in the case the branch has more than 20 commits on top of master fetch(branch: branch, depth: depth) - fetch(branch: 'master', depth: depth) + fetch(branch: 'master', depth: depth, remote: DEFAULT_CE_PROJECT_URL) merge_base_found? end @@ -201,10 +201,10 @@ module Gitlab raise "\n#{branch} is too far behind master, please rebase it!\n" unless success end - def fetch(branch:, depth:) + def fetch(branch:, depth:, remote: 'origin') step( "Fetching deeper...", - %W[git fetch --depth=#{depth} --prune origin +refs/heads/#{branch}:refs/remotes/origin/#{branch}] + %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/origin/#{branch}] ) do |output, status| raise "Fetch failed: #{output}" unless status.zero? end -- cgit v1.2.1 From f4c7fea613ad996d855180c9c25f3d0cdca5121a Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 15 Nov 2017 15:20:36 +0100 Subject: Fix dumping hashed storage based repository --- lib/backup/repository.rb | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 3ad09a1b421..b6d273b98c2 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -7,12 +7,16 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " + progress.print " * #{display_repo_path(project)} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - # Create namespace dir if missing - FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace + # Create namespace dir or hashed path if missing + if project.hashed_storage?(:repository) + FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) + else + FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace + end if empty_repo?(project) progress.puts "[SKIPPED]".color(:cyan) @@ -42,7 +46,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_repo) - progress.print " * #{wiki.full_path} ... " + progress.print " * #{display_repo_path(wiki)} ... " if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else @@ -71,7 +75,7 @@ module Backup end Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " + progress.print " * #{display_repo_path(project)} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -104,7 +108,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - progress.print " * #{wiki.full_path} ... " + progress.print " * #{display_repo_path(wiki)} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then @@ -185,14 +189,14 @@ module Backup def progress_warn(project, cmd, output) progress.puts "[WARNING] Executing #{cmd}".color(:orange) - progress.puts "Ignoring error on #{project.full_path} - #{output}".color(:orange) + progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange) end def empty_repo?(project_or_wiki) project_or_wiki.repository.expire_exists_cache # protect backups from stale cache project_or_wiki.repository.empty_repo? rescue => e - progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.full_path} - #{e.message}".color(:orange) + progress.puts "Ignoring repository error and continuing backing up project: #{display_repo_path(project_or_wiki)} - #{e.message}".color(:orange) false end @@ -204,5 +208,9 @@ module Backup def progress $progress end + + def display_repo_path(project) + project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path + end end end -- cgit v1.2.1 From 05c10c9bf77db2a9e37e61d1953ef0150289322d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 14 Nov 2017 18:55:00 +0100 Subject: Add total_time_spent to the `changes` hash in issuable Webhook payloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/hook_data/issue_builder.rb | 1 + lib/gitlab/hook_data/merge_request_builder.rb | 1 + 2 files changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index 196f2b6b34c..e29dd0d5b0e 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -28,6 +28,7 @@ module Gitlab SAFE_HOOK_RELATIONS = %i[ assignees labels + total_time_spent ].freeze attr_accessor :issue diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 503452c8ff3..ae9b68eb648 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -33,6 +33,7 @@ module Gitlab SAFE_HOOK_RELATIONS = %i[ assignee labels + total_time_spent ].freeze attr_accessor :merge_request -- cgit v1.2.1 From 44c3fb6e817b43f36f21c54b7d1503da45005d2e Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Mon, 13 Nov 2017 11:05:16 +0100 Subject: Add an attributes bag class as a GitalyClient helper --- lib/gitlab/gitaly_client/attributes_bag.rb | 31 ++++++++++++++++++++++++++++++ lib/gitlab/gitaly_client/diff.rb | 16 ++------------- lib/gitlab/gitaly_client/diff_stitcher.rb | 2 +- lib/gitlab/gitaly_client/wiki_file.rb | 12 ++---------- lib/gitlab/gitaly_client/wiki_page.rb | 10 +++------- 5 files changed, 39 insertions(+), 32 deletions(-) create mode 100644 lib/gitlab/gitaly_client/attributes_bag.rb (limited to 'lib') diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb new file mode 100644 index 00000000000..198a1de91c7 --- /dev/null +++ b/lib/gitlab/gitaly_client/attributes_bag.rb @@ -0,0 +1,31 @@ +module Gitlab + module GitalyClient + # This module expects an `ATTRS` const to be defined on the subclass + # See GitalyClient::WikiFile for an example + module AttributesBag + extend ActiveSupport::Concern + + included do + attr_accessor(*const_get(:ATTRS)) + end + + def initialize(params) + params = params.with_indifferent_access + + attributes.each do |attr| + instance_variable_set("@#{attr}", params[attr]) + end + end + + def ==(other) + attributes.all? do |field| + instance_variable_get("@#{field}") == other.instance_variable_get("@#{field}") + end + end + + def attributes + self.class.const_get(:ATTRS) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index 54df6304865..d98a0ce988f 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -1,21 +1,9 @@ module Gitlab module GitalyClient class Diff - FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze + ATTRS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze - attr_accessor(*FIELDS) - - def initialize(params) - params.each do |key, val| - public_send(:"#{key}=", val) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def ==(other) - FIELDS.all? do |field| - public_send(field) == other.public_send(field) # rubocop:disable GitlabSecurity/PublicSend - end - end + include AttributesBag end end end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb index 65d81dc5d46..da243ee2d1a 100644 --- a/lib/gitlab/gitaly_client/diff_stitcher.rb +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -12,7 +12,7 @@ module Gitlab @rpc_response.each do |diff_msg| if current_diff.nil? - diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) + diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::ATTRS) # gRPC uses frozen strings by default, and we need to have an unfrozen string as it # gets processed further down the line. So we unfreeze the first chunk of the patch # in case it's the only chunk we receive for this diff. diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb index a2e415864e6..47c60c92484 100644 --- a/lib/gitlab/gitaly_client/wiki_file.rb +++ b/lib/gitlab/gitaly_client/wiki_file.rb @@ -1,17 +1,9 @@ module Gitlab module GitalyClient class WikiFile - FIELDS = %i(name mime_type path raw_data).freeze + ATTRS = %i(name mime_type path raw_data).freeze - attr_accessor(*FIELDS) - - def initialize(params) - params = params.with_indifferent_access - - FIELDS.each do |field| - instance_variable_set("@#{field}", params[field]) - end - end + include AttributesBag end end end diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index 98d96fe6211..7339468e911 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -1,16 +1,12 @@ module Gitlab module GitalyClient class WikiPage - FIELDS = %i(title format url_path path name historical raw_data).freeze + ATTRS = %i(title format url_path path name historical raw_data).freeze - attr_accessor(*FIELDS) + include AttributesBag def initialize(params) - params = params.with_indifferent_access - - FIELDS.each do |field| - instance_variable_set("@#{field}", params[field]) - end + super # All gRPC strings in a response are frozen, so we get an unfrozen # version here so appending to `raw_data` doesn't blow up. -- cgit v1.2.1 From f4df4f9e3504f0f2fd7acf86087aa0e22177e981 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 16 Nov 2017 09:31:07 +0100 Subject: Update container repository path reference We should allow to use double underscore in the path, and it seems that our container repository path regexp was outdated. See https://github.com/docker/distribution/blob/master/reference/regexp.go --- lib/gitlab/regex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index bd677ec4bf3..2c7b8af83f2 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -25,7 +25,7 @@ module Gitlab # See https://github.com/docker/distribution/blob/master/reference/regexp.go. # def container_repository_name_regex - @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} + @container_repository_regex ||= %r{\A[a-z0-9]+((?:[._/]|__|[-])[a-z0-9]+)*\Z} end ## -- cgit v1.2.1 From 181cd299f9e06223e8338e93b1c318c671ccb1aa Mon Sep 17 00:00:00 2001 From: Jacopo Date: Tue, 14 Nov 2017 10:02:39 +0100 Subject: Adds Rubocop rule for line break after guard clause Adds a rubocop rule (with autocorrect) to ensure line break after guard clauses. --- lib/api/commits.rb | 2 ++ lib/api/helpers/custom_validators.rb | 1 + lib/api/helpers/runner.rb | 1 + lib/api/runners.rb | 4 ++++ lib/api/snippets.rb | 1 + lib/api/v3/commits.rb | 2 ++ lib/api/v3/runners.rb | 1 + lib/api/v3/snippets.rb | 2 ++ lib/banzai/object_renderer.rb | 1 + lib/banzai/querying.rb | 2 ++ lib/banzai/reference_parser/user_parser.rb | 1 + lib/banzai/renderer.rb | 2 ++ lib/declarative_policy.rb | 2 ++ lib/declarative_policy/base.rb | 2 ++ lib/declarative_policy/cache.rb | 2 ++ lib/declarative_policy/rule.rb | 5 +++++ lib/declarative_policy/runner.rb | 1 + lib/file_size_validator.rb | 1 + lib/gitlab/changes_list.rb | 1 + lib/gitlab/ci/build/artifacts/metadata.rb | 1 + lib/gitlab/ci/build/artifacts/metadata/entry.rb | 3 +++ lib/gitlab/ci/build/image.rb | 1 + lib/gitlab/ci/config/entry/image.rb | 1 + lib/gitlab/ci/config/entry/validators.rb | 1 + lib/gitlab/daemon.rb | 1 + lib/gitlab/diff/inline_diff.rb | 1 + lib/gitlab/diff/parser.rb | 1 + lib/gitlab/diff/position.rb | 1 + lib/gitlab/email/handler/unsubscribe_handler.rb | 1 + lib/gitlab/fogbugz_import/client.rb | 1 + lib/gitlab/fogbugz_import/importer.rb | 2 ++ lib/gitlab/git/blob.rb | 1 + lib/gitlab/git/repository.rb | 3 +++ lib/gitlab/gitaly_client/ref_service.rb | 1 + lib/gitlab/gitaly_client/wiki_service.rb | 1 + lib/gitlab/gitlab_import/client.rb | 1 + lib/gitlab/kubernetes/namespace.rb | 1 + lib/gitlab/ldap/authentication.rb | 1 + lib/gitlab/legacy_github_import/importer.rb | 1 + lib/gitlab/metrics/samplers/ruby_sampler.rb | 1 + lib/gitlab/metrics/subscribers/active_record.rb | 1 + lib/gitlab/middleware/go.rb | 1 + lib/gitlab/optimistic_locking.rb | 1 + lib/gitlab/saml/user.rb | 1 + lib/gitlab/shell.rb | 1 + lib/gitlab/string_range_marker.rb | 1 + lib/gitlab/template/finders/repo_template_finder.rb | 1 + lib/gitlab/url_sanitizer.rb | 1 + lib/gitlab/visibility_level.rb | 1 + lib/gitlab/workhorse.rb | 1 + lib/haml_lint/inline_javascript.rb | 1 + lib/system_check/simple_executor.rb | 1 + lib/tasks/gitlab/cleanup.rake | 2 ++ 53 files changed, 74 insertions(+) (limited to 'lib') diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 2bc4039b019..38e05074353 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -180,10 +180,12 @@ module API if params[:path] commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index 0a8f3073a50..dd4f6c41131 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -4,6 +4,7 @@ module API class Absence < Grape::Validations::Base def validate_param!(attr_name, params) return if params.respond_to?(:key?) && !params.key?(attr_name) + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 282af32ca94..2cae53dba53 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -14,6 +14,7 @@ module API def get_runner_version_from_params return unless params['info'].present? + attributes_for_keys(%w(name version revision platform architecture), params['info']) end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index d3559ef71be..e816fcdd928 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -165,17 +165,20 @@ module API def authenticate_show_runner!(runner) return if runner.is_shared || current_user.admin? + forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_update_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_delete_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) @@ -185,6 +188,7 @@ module API forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner is locked") if runner.locked? return if current_user.admin? + forbidden!("No access granted") unless user_can_access_runner?(runner) end diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 00eb7c60f16..c736cc32021 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -95,6 +95,7 @@ module API put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet attrs = declared_params(include_missing: false).merge(request: request, api: true) diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index be360fbfc0c..0ef26aa696a 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -169,10 +169,12 @@ module API if params[:path] commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb index faa265f3314..c6d9957d452 100644 --- a/lib/api/v3/runners.rb +++ b/lib/api/v3/runners.rb @@ -51,6 +51,7 @@ module API helpers do def authenticate_delete_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 0762fc02d70..126ec72248e 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -91,6 +91,7 @@ module API put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet attrs = declared_params(include_missing: false) @@ -113,6 +114,7 @@ module API delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :destroy_personal_snippet, snippet snippet.destroy no_content! diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9bb8ed913d8..ecb3affbba5 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -86,6 +86,7 @@ module Banzai def save_options return {} unless base_context[:xhtml] + { save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML } end end diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb index fb2faae02bc..a19a05e8c0d 100644 --- a/lib/banzai/querying.rb +++ b/lib/banzai/querying.rb @@ -52,8 +52,10 @@ module Banzai children.each do |child| next if child.text.blank? + node = nodes.shift break unless node == child + filtered_nodes << node end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 4d336068861..8932d4f2905 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -31,6 +31,7 @@ module Banzai nodes.each do |node| if node.has_attribute?(group_attr) next unless can_read_group_reference?(node, user, groups) + visible << node elsif can_read_project_reference?(node) visible << node diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 5cb9adf52b0..0050295eeda 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -149,6 +149,7 @@ module Banzai def self.full_cache_key(cache_key, pipeline_name) return unless cache_key + ["banzai", *cache_key, pipeline_name || :full] end @@ -157,6 +158,7 @@ module Banzai # method. def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key + Rails.cache.__send__(:expanded_key, full_cache_key(cache_key, pipeline_name)) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index ae65653645b..b1949d693ad 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -30,6 +30,7 @@ module DeclarativePolicy policy_class = class_for_class(subject.class) raise "no policy for #{subject.class.name}" if policy_class.nil? + policy_class end @@ -84,6 +85,7 @@ module DeclarativePolicy while subject.respond_to?(:declarative_policy_delegate) raise ArgumentError, "circular delegations" if seen.include?(subject.object_id) + seen << subject.object_id subject = subject.declarative_policy_delegate end diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index b028169f500..47542194497 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -276,6 +276,7 @@ module DeclarativePolicy # boolean `false` def cache(key, &b) return @cache[key] if cached?(key) + @cache[key] = yield end @@ -291,6 +292,7 @@ module DeclarativePolicy @_conditions[name] ||= begin raise "invalid condition #{name}" unless self.class.conditions.key?(name) + ManifestCondition.new(self.class.conditions[name], self) end end diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb index 0804edba016..780d8f707bd 100644 --- a/lib/declarative_policy/cache.rb +++ b/lib/declarative_policy/cache.rb @@ -3,6 +3,7 @@ module DeclarativePolicy class << self def user_key(user) return '' if user.nil? + id_for(user) end @@ -15,6 +16,7 @@ module DeclarativePolicy def subject_key(subject) return '' if subject.nil? return subject.inspect if subject.is_a?(Symbol) + "#{subject.class.name}:#{id_for(subject)}" end diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index 7cfa82a9a9f..e309244a3b3 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -83,6 +83,7 @@ module DeclarativePolicy def cached_pass?(context) condition = context.condition(@name) return nil unless condition.cached? + condition.pass? end @@ -109,6 +110,7 @@ module DeclarativePolicy def delegated_context(context) policy = context.delegated_policies[@delegate_name] raise MissingDelegate if policy.nil? + policy end @@ -121,6 +123,7 @@ module DeclarativePolicy def cached_pass?(context) condition = delegated_context(context).condition(@name) return nil unless condition.cached? + condition.pass? rescue MissingDelegate false @@ -157,6 +160,7 @@ module DeclarativePolicy def cached_pass?(context) runner = context.runner(@ability) return nil unless runner.cached? + runner.pass? end @@ -258,6 +262,7 @@ module DeclarativePolicy def score(context) return 0 unless cached_pass?(context).nil? + @rules.map { |r| r.score(context) }.inject(0, :+) end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 45ff2ef9ced..77c91817382 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -43,6 +43,7 @@ module DeclarativePolicy # used by Rule::Ability. See #steps_by_score def score return 0 if cached? + steps.map(&:score).inject(0, :+) end diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index de391de9059..69d981e8be9 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -8,6 +8,7 @@ class FileSizeValidator < ActiveModel::EachValidator def initialize(options) if range = (options.delete(:in) || options.delete(:within)) raise ArgumentError, ":in and :within must be a Range" unless range.is_a?(Range) + options[:minimum], options[:maximum] = range.begin, range.end options[:maximum] -= 1 if range.exclude_end? end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index 5b32fca00a4..9c9e6668e6f 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -16,6 +16,7 @@ module Gitlab @changes ||= begin @raw_changes.map do |change| next if change.blank? + oldrev, newrev, ref = change.strip.split(' ') { oldrev: oldrev, newrev: newrev, ref: ref } end.compact diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index a788fb3fcbc..0bbd60d8ffe 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -98,6 +98,7 @@ module Gitlab def read_string(gz) string_size = read_uint32(gz) return nil unless string_size + gz.read(string_size) end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 22941d48edf..5b2f09e03ea 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -43,6 +43,7 @@ module Gitlab def parent return nil unless has_parent? + self.class.new(@path.to_s.chomp(basename), @entries) end @@ -64,6 +65,7 @@ module Gitlab def directories(opts = {}) return [] unless directory? + dirs = children.select(&:directory?) return dirs unless has_parent? && opts[:parent] @@ -74,6 +76,7 @@ module Gitlab def files return [] unless directory? + children.select(&:file?) end diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index b88b2e36d53..c811f88f483 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -8,6 +8,7 @@ module Gitlab def from_image(job) image = Gitlab::Ci::Build::Image.new(job.options[:image]) return unless image.valid? + image end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 6555c589173..2844be80a84 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -37,6 +37,7 @@ module Gitlab def value return { name: @config } if string? return @config if hash? + {} end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index 0159179f0a9..eb606b57667 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -111,6 +111,7 @@ module Gitlab def validate_string_or_regexp(value) return false unless value.is_a?(String) return validate_regexp(value) if look_like_regexp?(value) + true end end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index f07fd1dfdda..633de9f9776 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -2,6 +2,7 @@ module Gitlab class Daemon def self.initialize_instance(*args) raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) Kernel.at_exit(&@instance.method(:stop)) @instance diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 55708d42161..2d7b57120a6 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -102,6 +102,7 @@ module Gitlab new_char = b[pos] break if old_char != new_char + length += 1 end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7dc9cc7c281..8302f30a0a2 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -30,6 +30,7 @@ module Gitlab line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 next if line_old <= 1 && line_new <= 1 # top of file + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index ccfb908bcca..690b27cde81 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -125,6 +125,7 @@ module Gitlab def find_diff_file(repository) return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) + comparison.diffs(paths: paths, expanded: true).diff_files.first end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 5894384da5d..ea80e21532e 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -16,6 +16,7 @@ module Gitlab noteable = sent_notification.noteable raise NoteableNotFoundError unless noteable + noteable.unsubscribe(sent_notification.recipient) end diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb index 2152182b37f..acb000e3e23 100644 --- a/lib/gitlab/fogbugz_import/client.rb +++ b/lib/gitlab/fogbugz_import/client.rb @@ -45,6 +45,7 @@ module Gitlab project_name = repo(project_id).name res = @api.command(:search, q: "project:'#{project_name}'", cols: 'ixPersonAssignedTo,ixPersonOpenedBy,ixPersonClosedBy,sStatus,sPriority,sCategory,fOpen,sTitle,sLatestTextSummary,dtOpened,dtClosed,dtResolved,dtLastUpdated,events') return [] unless res['cases']['count'].to_i > 0 + res['cases']['case'] end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 3dcee681c72..5e426b13ade 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -18,6 +18,7 @@ module Gitlab def execute return true unless repo.valid? + client = Gitlab::FogbugzImport::Client.new(token: fb_session[:token], uri: fb_session[:uri]) @cases = client.cases(@repo.id.to_i) @@ -206,6 +207,7 @@ module Gitlab def format_content(raw_content) return raw_content if raw_content.nil? + linkify_issues(escape_for_markdown(raw_content)) end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index cc6c7609ec7..bd5039fb87e 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -102,6 +102,7 @@ module Gitlab if path_arr.size > 1 return nil unless entry[:type] == :tree + path_arr.shift find_entry_by_path(repository, entry[:oid], path_arr.join('/')) else diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index cfb88a0c12b..aad8464dff4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1376,6 +1376,7 @@ module Gitlab end return nil unless tmp_entry.type == :tree + tmp_entry = tmp_entry[dir] end end @@ -1496,6 +1497,7 @@ module Gitlab # Ref names must start with `refs/`. def rugged_ref_exists?(ref_name) raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/') + rugged.references.exist?(ref_name) rescue Rugged::ReferenceError false @@ -1562,6 +1564,7 @@ module Gitlab Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index b0c73395cb1..e8a2215959d 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -137,6 +137,7 @@ module Gitlab enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 8f05f40365e..c8f065f5881 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -94,6 +94,7 @@ module Gitlab page, version = wiki_page_from_iterator(response) { |message| message.end_of_page } break unless page && version + pages << [page, version] end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index f1007daab5d..075b3982608 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -65,6 +65,7 @@ module Gitlab y << item end break if items.empty? || items.size < per_page + page += 1 end end diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb index c8479fbc0e8..fbbddb7bffa 100644 --- a/lib/gitlab/kubernetes/namespace.rb +++ b/lib/gitlab/kubernetes/namespace.rb @@ -12,6 +12,7 @@ module Gitlab @client.get_namespace(name) rescue ::KubeException => ke raise ke unless ke.error_code == 404 + false end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index ed1de73f8c6..7274d1c3b43 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -62,6 +62,7 @@ module Gitlab def user return nil unless ldap_user + Gitlab::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) end end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 12c968805f5..4d096e5a741 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -15,6 +15,7 @@ module Gitlab def client return @client if defined?(@client) + unless credentials raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 8b5a60e6b8b..436a9e9550d 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -96,6 +96,7 @@ module Gitlab def worker_label return {} unless defined?(Unicorn::Worker) + worker_no = ::Prometheus::Client::Support::Unicorn.worker_id if worker_no diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 064299f40c8..ead1acb8d44 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -7,6 +7,7 @@ module Gitlab def sql(event) return unless current_transaction + metric_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) current_transaction.increment(:sql_duration, event.duration, false) diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index cfc6b2a2029..d87e3a17914 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -66,6 +66,7 @@ module Gitlab project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX) return unless project_path_match + path = project_path_match[1] # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`. diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 962ff4d3985..1d9a5d1a20a 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -11,6 +11,7 @@ module Gitlab rescue ActiveRecord::StaleObjectError retries -= 1 raise unless retries >= 0 + subject.reload end end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index e0a9d1dee77..d8faf7aad8c 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -28,6 +28,7 @@ module Gitlab def changed? return true unless gl_user + gl_user.changed? || gl_user.identities.any?(&:changed?) end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index a37112ae5c4..dc0184e4ad9 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -368,6 +368,7 @@ module Gitlab output, status = gitlab_shell_fast_execute_helper(cmd, vars) raise Error, output unless status.zero? + true end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 11aeec1ebfa..f9faa134206 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -90,6 +90,7 @@ module Gitlab # Takes an array of integers, and returns an array of ranges covering the same integers def collapse_ranges(positions) return [] if positions.empty? + ranges = [] start = prev = positions[0] diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index cb7957e2af9..33f07fa0120 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -18,6 +18,7 @@ module Gitlab def read(path) blob = @repository.blob_at(@commit.id, path) if @commit raise FileNotFoundError if blob.nil? + blob.data end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 1caa791c1be..59331c827af 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -70,6 +70,7 @@ module Gitlab def generate_full_url return @url unless valid_credentials? + @full_url = @url.dup @full_url.password = credentials[:password] if credentials[:password].present? diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index c60bd91ea6e..11472ce6cce 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -99,6 +99,7 @@ module Gitlab def level_value(level) return level.to_i if level.to_i.to_s == level.to_s && string_options.key(level.to_i) + string_options[level] || PRIVATE end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e1219df1b25..864a9e04888 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -174,6 +174,7 @@ module Gitlab @secret ||= begin bytes = Base64.strict_decode64(File.read(secret_path).chomp) raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH + bytes end end diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb index 05668c69006..f5485eb89fa 100644 --- a/lib/haml_lint/inline_javascript.rb +++ b/lib/haml_lint/inline_javascript.rb @@ -9,6 +9,7 @@ unless Rails.env.production? def visit_filter(node) return unless node.filter_type == 'javascript' + record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') end end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 00221f77cf4..8b145fb4511 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -24,6 +24,7 @@ module SystemCheck # @param [BaseCheck] check class def <<(check) raise ArgumentError unless check.is_a?(Class) && check < BaseCheck + @checks << check end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 8ae1b6a626a..91c74bfb6b4 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -60,6 +60,7 @@ namespace :gitlab do .chomp('.git') .chomp('.wiki') next if Project.find_by_full_path(repo_with_namespace) + new_path = path + move_suffix puts path.inspect + ' -> ' + new_path.inspect File.rename(path, new_path) @@ -75,6 +76,7 @@ namespace :gitlab do User.find_each do |user| next unless user.ldap_user? + print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." if Gitlab::LDAP::Access.allowed?(user) puts " [OK]".color(:green) -- cgit v1.2.1 From 9f921b73f2796554f79a8b730999ac884daf4a19 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Thu, 16 Nov 2017 17:12:45 +0000 Subject: Don't add a trailing slash in group redirects Because we ignored the format, a request to `/groups/foo/labels.json` would redirect to `/groups/foo/-/labels/.json`. But really, it's worse than that, because unless the request contained a trailing slash, we shouldn't add one. Now, we only _keep_ a trailing slash, but don't _add_ one. --- lib/gitlab/routing.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 910533076b0..2c994536060 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -46,10 +46,10 @@ module Gitlab # Only replace the last occurence of `path`. # # `request.fullpath` includes the querystring - path = request.path.sub(%r{/#{path}/*(?!.*#{path})}, "/-/#{path}/") - path << "?#{request.query_string}" if request.query_string.present? + new_path = request.path.sub(%r{/#{path}(/*)(?!.*#{path})}, "/-/#{path}\\1") + new_path << "?#{request.query_string}" if request.query_string.present? - path + new_path end paths.each do |path| -- cgit v1.2.1 From dc9266fbeacd24446b52e4dad328c8286be40b31 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 15 Sep 2017 10:31:32 -0700 Subject: Add request throttles --- lib/gitlab/auth.rb | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index cbbc51db99e..8f39885c608 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -82,6 +82,36 @@ module Gitlab end end + # request may be Rack::Attack::Request which is just a Rack::Request, so + # we cannot use ActionDispatch::Request methods. + def find_user_by_private_token(request) + token = request.params['private_token'].presence || request.env['HTTP_PRIVATE_TOKEN'].presence + + return unless token.present? + + User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) + end + + # request may be Rack::Attack::Request which is just a Rack::Request, so + # we cannot use ActionDispatch::Request methods. + def find_user_by_rss_token(request) + return unless request.params['format'] == 'atom' + + token = request.params['rss_token'].presence + + return unless token.present? + + User.find_by_rss_token(token) + end + + def find_session_user(request) + request.env['warden']&.authenticate + end + + def find_sessionless_user(request) + find_user_by_private_token(request) || find_user_by_rss_token(request) + end + private def service_request_check(login, password, project) -- cgit v1.2.1 From 43a682ccaa694d2a14f3d639d66708057859a628 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 13 Oct 2017 17:05:18 -0700 Subject: Fix OAuth API and RSS rate limiting --- lib/gitlab/auth.rb | 30 --------------- lib/gitlab/auth/request_authenticator.rb | 64 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 30 deletions(-) create mode 100644 lib/gitlab/auth/request_authenticator.rb (limited to 'lib') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8f39885c608..cbbc51db99e 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -82,36 +82,6 @@ module Gitlab end end - # request may be Rack::Attack::Request which is just a Rack::Request, so - # we cannot use ActionDispatch::Request methods. - def find_user_by_private_token(request) - token = request.params['private_token'].presence || request.env['HTTP_PRIVATE_TOKEN'].presence - - return unless token.present? - - User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) - end - - # request may be Rack::Attack::Request which is just a Rack::Request, so - # we cannot use ActionDispatch::Request methods. - def find_user_by_rss_token(request) - return unless request.params['format'] == 'atom' - - token = request.params['rss_token'].presence - - return unless token.present? - - User.find_by_rss_token(token) - end - - def find_session_user(request) - request.env['warden']&.authenticate - end - - def find_sessionless_user(request) - find_user_by_private_token(request) || find_user_by_rss_token(request) - end - private def service_request_check(login, password, project) diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb new file mode 100644 index 00000000000..d3da4cc2d2b --- /dev/null +++ b/lib/gitlab/auth/request_authenticator.rb @@ -0,0 +1,64 @@ +# Use for authentication only, in particular for Rack::Attack. +# Does not perform authorization of scopes, etc. +module Gitlab + module Auth + class RequestAuthenticator + def initialize(request) + @request = request + end + + def user + find_sessionless_user || find_session_user + end + + def find_sessionless_user + find_user_by_private_token || find_user_by_rss_token || find_user_by_oauth_token + end + + private + + def find_session_user + @request.env['warden']&.authenticate if verified_request? + end + + # request may be Rack::Attack::Request which is just a Rack::Request, so + # we cannot use ActionDispatch::Request methods. + def find_user_by_private_token + token = @request.params['private_token'].presence || @request.env['HTTP_PRIVATE_TOKEN'].presence + return unless token.present? + + User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) + end + + # request may be Rack::Attack::Request which is just a Rack::Request, so + # we cannot use ActionDispatch::Request methods. + def find_user_by_rss_token + return unless @request.path.ends_with?('atom') || @request.env['HTTP_ACCEPT'] == 'application/atom+xml' + + token = @request.params['rss_token'].presence + return unless token.present? + + User.find_by_rss_token(token) + end + + def find_user_by_oauth_token + access_token = find_oauth_access_token + access_token&.user + end + + def find_oauth_access_token + token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) + OauthAccessToken.by_token(token) if token + end + + def doorkeeper_request + ActionDispatch::Request.new(@request.env) + end + + # Check if the request is GET/HEAD, or if CSRF token is valid. + def verified_request? + Gitlab::RequestForgeryProtection.verified?(@request.env) + end + end + end +end -- cgit v1.2.1 From 4e5a97d4f3cc8b17728fe116fc24c043a03f38c6 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sat, 21 Oct 2017 22:10:03 +0300 Subject: Refactor with ActionDispatch::Request --- lib/gitlab/auth/request_authenticator.rb | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index d3da4cc2d2b..999104f91f5 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -4,7 +4,7 @@ module Gitlab module Auth class RequestAuthenticator def initialize(request) - @request = request + @request = ensure_action_dispatch_request(request) end def user @@ -21,21 +21,17 @@ module Gitlab @request.env['warden']&.authenticate if verified_request? end - # request may be Rack::Attack::Request which is just a Rack::Request, so - # we cannot use ActionDispatch::Request methods. def find_user_by_private_token - token = @request.params['private_token'].presence || @request.env['HTTP_PRIVATE_TOKEN'].presence + token = @request.params[:private_token].presence || @request.headers['PRIVATE-TOKEN'].presence return unless token.present? User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) end - # request may be Rack::Attack::Request which is just a Rack::Request, so - # we cannot use ActionDispatch::Request methods. def find_user_by_rss_token - return unless @request.path.ends_with?('atom') || @request.env['HTTP_ACCEPT'] == 'application/atom+xml' + return unless @request.path.ends_with?('atom') || @request.format == 'atom' - token = @request.params['rss_token'].presence + token = @request.params[:rss_token].presence return unless token.present? User.find_by_rss_token(token) @@ -47,18 +43,20 @@ module Gitlab end def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) + token = Doorkeeper::OAuth::Token.from_request(@request, *Doorkeeper.configuration.access_token_methods) OauthAccessToken.by_token(token) if token end - def doorkeeper_request - ActionDispatch::Request.new(@request.env) - end - # Check if the request is GET/HEAD, or if CSRF token is valid. def verified_request? Gitlab::RequestForgeryProtection.verified?(@request.env) end + + def ensure_action_dispatch_request(request) + return request if request.is_a?(ActionDispatch::Request) + + ActionDispatch::Request.new(request.env) + end end end end -- cgit v1.2.1 From d948e6791300b14d18b95881290ccfcba7928ea0 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Tue, 7 Nov 2017 10:52:05 +0100 Subject: First refactor --- lib/api/api_guard.rb | 58 +++++++------------- lib/gitlab/auth/request_authenticator.rb | 47 ++-------------- lib/gitlab/auth/user_auth_finders.rb | 92 ++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 82 deletions(-) create mode 100644 lib/gitlab/auth/user_auth_finders.rb (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index c1c0d344917..0a93e71858e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -74,43 +74,27 @@ module API private - def find_user_from_access_token - return unless access_token - - validate_access_token! - - access_token.user || raise(UnauthorizedError) - end - - # Check the Rails session for valid authentication details - def find_user_from_warden - warden.try(:authenticate) if verified_request? - end - - def warden - env['warden'] - end - - # Check if the request is GET/HEAD, or if CSRF token is valid. - def verified_request? - Gitlab::RequestForgeryProtection.verified?(env) - end - - def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) - return unless token - - # Expiration, revocation and scopes are verified in `find_user_by_access_token` - access_token = OauthAccessToken.by_token(token) - raise UnauthorizedError unless access_token - - access_token.revoke_previous_refresh_token! - access_token + def raise_unauthorized_error! + raise UnauthorizedError end - def find_personal_access_token - token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - return unless token.present? + # If token is presented and valid, then it sets @current_user. + # + # If the token does not have sufficient scopes to cover the requred scopes, + # then it raises InsufficientScopeError. + # + # If the token is expired, then it raises ExpiredError. + # + # If the token is revoked, then it raises RevokedError. + # + # If the token is not found (nil), then it returns nil + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + def find_user_by_access_token(access_token) + scopes = scopes_registered_for_endpoint # Expiration, revocation and scopes are verified in `find_user_by_access_token` access_token = PersonalAccessToken.find_by(token: token) @@ -119,10 +103,6 @@ module API access_token end - def doorkeeper_request - @doorkeeper_request ||= ActionDispatch::Request.new(env) - end - # An array of scopes that were registered (using `allow_access_with_scope`) # for the current endpoint class. It also returns scopes registered on # `API::API`, since these are meant to apply to all API routes. diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 999104f91f5..123a39dea75 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -3,6 +3,10 @@ module Gitlab module Auth class RequestAuthenticator + include UserAuthFinders + + attr_reader :request + def initialize(request) @request = ensure_action_dispatch_request(request) end @@ -14,49 +18,6 @@ module Gitlab def find_sessionless_user find_user_by_private_token || find_user_by_rss_token || find_user_by_oauth_token end - - private - - def find_session_user - @request.env['warden']&.authenticate if verified_request? - end - - def find_user_by_private_token - token = @request.params[:private_token].presence || @request.headers['PRIVATE-TOKEN'].presence - return unless token.present? - - User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) - end - - def find_user_by_rss_token - return unless @request.path.ends_with?('atom') || @request.format == 'atom' - - token = @request.params[:rss_token].presence - return unless token.present? - - User.find_by_rss_token(token) - end - - def find_user_by_oauth_token - access_token = find_oauth_access_token - access_token&.user - end - - def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(@request, *Doorkeeper.configuration.access_token_methods) - OauthAccessToken.by_token(token) if token - end - - # Check if the request is GET/HEAD, or if CSRF token is valid. - def verified_request? - Gitlab::RequestForgeryProtection.verified?(@request.env) - end - - def ensure_action_dispatch_request(request) - return request if request.is_a?(ActionDispatch::Request) - - ActionDispatch::Request.new(request.env) - end end end end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb new file mode 100644 index 00000000000..2f4aeff14ac --- /dev/null +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -0,0 +1,92 @@ +module Gitlab + module Auth + module UserAuthFinders + # Check the Rails session for valid authentication details + def find_session_user + request.env['warden']&.authenticate if verified_request? + end + + def find_user_by_private_token + token = private_token + return unless token.present? + + user = + find_user_by_authentication_token(token) || + find_user_by_personal_access_token(token) + + raise_unauthorized_error! unless user + + user + end + + def private_token + request.params[:private_token].presence || + request.headers['PRIVATE-TOKEN'].presence + end + + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end + + def find_user_by_personal_access_token(token_string) + access_token = PersonalAccessToken.find_by_token(token_string) + return unless access_token + + find_user_by_access_token(access_token) + end + + def find_user_by_rss_token + return unless request.path.ends_with?('atom') || request.format.atom? + + token = request.params[:rss_token].presence + return unless token.present? + + user = User.find_by_rss_token(token) + raise_unauthorized_error! unless user + + user + end + + def find_user_by_oauth_token + access_token = find_oauth_access_token + + return unless access_token + + find_user_by_access_token(access_token) + end + + def find_oauth_access_token + return @oauth_access_token if defined?(@oauth_access_token) + + current_request = ensure_action_dispatch_request(request) + token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + return @oauth_access_token = nil unless token + + @oauth_access_token = OauthAccessToken.by_token(token) + raise_unauthorized_error! unless @oauth_access_token + + @oauth_access_token.revoke_previous_refresh_token! + @oauth_access_token + end + + def find_user_by_access_token(access_token) + access_token&.user + end + + # Check if the request is GET/HEAD, or if CSRF token is valid. + def verified_request? + Gitlab::RequestForgeryProtection.verified?(request.env) + end + + def ensure_action_dispatch_request(request) + return request if request.is_a?(ActionDispatch::Request) + + ActionDispatch::Request.new(request.env) + end + + def raise_unauthorized_error! + return nil + end + end + end +end -- cgit v1.2.1 From 470b5dc32633cd4ec873e655ac6a70011c835e17 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Tue, 7 Nov 2017 16:13:00 +0100 Subject: Updated refactor and pushing to see if test fails --- lib/api/api_guard.rb | 2 -- lib/gitlab/auth/user_auth_finders.rb | 34 ++++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 0a93e71858e..66ad2b77f75 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -72,8 +72,6 @@ module API end end - private - def raise_unauthorized_error! raise UnauthorizedError end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 2f4aeff14ac..d1f5bf84873 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -19,22 +19,6 @@ module Gitlab user end - def private_token - request.params[:private_token].presence || - request.headers['PRIVATE-TOKEN'].presence - end - - def find_user_by_authentication_token(token_string) - User.find_by_authentication_token(token_string) - end - - def find_user_by_personal_access_token(token_string) - access_token = PersonalAccessToken.find_by_token(token_string) - return unless access_token - - find_user_by_access_token(access_token) - end - def find_user_by_rss_token return unless request.path.ends_with?('atom') || request.format.atom? @@ -55,6 +39,24 @@ module Gitlab find_user_by_access_token(access_token) end + private + + def private_token + request.params[:private_token].presence || + request.headers['PRIVATE-TOKEN'].presence + end + + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end + + def find_user_by_personal_access_token(token_string) + access_token = PersonalAccessToken.find_by_token(token_string) + return unless access_token + + find_user_by_access_token(access_token) + end + def find_oauth_access_token return @oauth_access_token if defined?(@oauth_access_token) -- cgit v1.2.1 From 41ebd06ddc837c80ba6ca95c6d5fea2b76cef8d2 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Tue, 7 Nov 2017 19:17:41 +0100 Subject: Some fixes after rebase --- lib/api/api_guard.rb | 31 +++---------- lib/gitlab/auth/request_authenticator.rb | 4 +- lib/gitlab/auth/user_auth_finders.rb | 78 +++++++++++++------------------- 3 files changed, 41 insertions(+), 72 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 66ad2b77f75..9ada2d5ebb1 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -72,33 +72,16 @@ module API end end - def raise_unauthorized_error! - raise UnauthorizedError - end + private - # If token is presented and valid, then it sets @current_user. - # - # If the token does not have sufficient scopes to cover the requred scopes, - # then it raises InsufficientScopeError. - # - # If the token is expired, then it raises ExpiredError. - # - # If the token is revoked, then it raises RevokedError. - # - # If the token is not found (nil), then it returns nil - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - def find_user_by_access_token(access_token) - scopes = scopes_registered_for_endpoint + def handle_return_value!(value, &block) + raise UnauthorizedError unless value - # Expiration, revocation and scopes are verified in `find_user_by_access_token` - access_token = PersonalAccessToken.find_by(token: token) - raise UnauthorizedError unless access_token + block_given? ? yield(value) : value + end - access_token + def private_token + params[PRIVATE_TOKEN_PARAM].presence || env[PRIVATE_TOKEN_HEADER].presence end # An array of scopes that were registered (using `allow_access_with_scope`) diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 123a39dea75..eb16701bad5 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -12,11 +12,11 @@ module Gitlab end def user - find_sessionless_user || find_session_user + find_sessionless_user || find_user_from_warden end def find_sessionless_user - find_user_by_private_token || find_user_by_rss_token || find_user_by_oauth_token + find_user_from_access_token || find_user_by_rss_token end end end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index d1f5bf84873..93f3cae0a95 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -2,77 +2,67 @@ module Gitlab module Auth module UserAuthFinders # Check the Rails session for valid authentication details - def find_session_user + def find_user_from_warden request.env['warden']&.authenticate if verified_request? end - def find_user_by_private_token - token = private_token - return unless token.present? - - user = - find_user_by_authentication_token(token) || - find_user_by_personal_access_token(token) + def find_user_by_rss_token + return unless request.format.atom? - raise_unauthorized_error! unless user + token = request.params[:rss_token].presence + return unless token.present? - user + handle_return_value!(User.find_by_rss_token(token)) end - def find_user_by_rss_token - return unless request.path.ends_with?('atom') || request.format.atom? + def find_user_from_access_token + return unless access_token - token = request.params[:rss_token].presence - return unless token.present? + validate_access_token! - user = User.find_by_rss_token(token) - raise_unauthorized_error! unless user + handle_return_value!(access_token&.user) + end - user + def validate_access_token!(scopes: []) end - def find_user_by_oauth_token - access_token = find_oauth_access_token + private - return unless access_token + def handle_return_value!(value, &block) + return unless value - find_user_by_access_token(access_token) + block_given? ? yield(value) : value end - private + def access_token + return @access_token if defined?(@access_token) + + @access_token = find_oauth_access_token || find_personal_access_token + end def private_token request.params[:private_token].presence || request.headers['PRIVATE-TOKEN'].presence end - def find_user_by_authentication_token(token_string) - User.find_by_authentication_token(token_string) - end - - def find_user_by_personal_access_token(token_string) - access_token = PersonalAccessToken.find_by_token(token_string) - return unless access_token + def find_personal_access_token + token = private_token.to_s + return unless token.present? - find_user_by_access_token(access_token) + # Expiration, revocation and scopes are verified in `validate_access_token!` + handle_return_value!(PersonalAccessToken.find_by(token: token)) end def find_oauth_access_token - return @oauth_access_token if defined?(@oauth_access_token) - current_request = ensure_action_dispatch_request(request) token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) - return @oauth_access_token = nil unless token - - @oauth_access_token = OauthAccessToken.by_token(token) - raise_unauthorized_error! unless @oauth_access_token - - @oauth_access_token.revoke_previous_refresh_token! - @oauth_access_token - end + return unless token - def find_user_by_access_token(access_token) - access_token&.user + # Expiration, revocation and scopes are verified in `validate_access_token!` + handle_return_value!(OauthAccessToken.by_token(token)) do |oauth_token| + oauth_token.revoke_previous_refresh_token! + oauth_token + end end # Check if the request is GET/HEAD, or if CSRF token is valid. @@ -85,10 +75,6 @@ module Gitlab ActionDispatch::Request.new(request.env) end - - def raise_unauthorized_error! - return nil - end end end end -- cgit v1.2.1 From 374179a97042da3a4d5312afcdb0dc90a44634f0 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Wed, 8 Nov 2017 10:13:22 +0100 Subject: Removing private token --- lib/api/api_guard.rb | 7 +------ lib/gitlab/auth/user_auth_finders.rb | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 9ada2d5ebb1..9c68830ae34 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -45,6 +45,7 @@ module API include Gitlab::Utils::StrongMemoize def find_current_user! + set_raise_unauthorized_error user = find_user_from_access_token || find_user_from_warden return unless user @@ -74,12 +75,6 @@ module API private - def handle_return_value!(value, &block) - raise UnauthorizedError unless value - - block_given? ? yield(value) : value - end - def private_token params[PRIVATE_TOKEN_PARAM].presence || env[PRIVATE_TOKEN_HEADER].presence end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 93f3cae0a95..86f1c13d4b8 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -29,7 +29,9 @@ module Gitlab private def handle_return_value!(value, &block) - return unless value + unless value + raise_unauthorized_error? ? raise_unauthorized_error! : return + end block_given? ? yield(value) : value end @@ -75,6 +77,18 @@ module Gitlab ActionDispatch::Request.new(request.env) end + + def raise_unauthorized_error? + defined?(@raise_unauthorized_error) ? @raise_unauthorized_error : false + end + + def set_raise_unauthorized_error + @raise_unauthorized_error = true + end + + def raise_unauthorized_error! + raise API::APIGuard::UnauthorizedError + end end end end -- cgit v1.2.1 From aecc3eb0809c4436a57f5ecdd88def58e704205d Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Wed, 8 Nov 2017 19:41:07 +0100 Subject: Applied some code review comments --- lib/api/api_guard.rb | 5 ---- lib/gitlab/auth/request_authenticator.rb | 8 ++++-- lib/gitlab/auth/user_auth_finders.rb | 45 ++++++++++++++++---------------- 3 files changed, 28 insertions(+), 30 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 9c68830ae34..01e15ffee84 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -45,7 +45,6 @@ module API include Gitlab::Utils::StrongMemoize def find_current_user! - set_raise_unauthorized_error user = find_user_from_access_token || find_user_from_warden return unless user @@ -75,10 +74,6 @@ module API private - def private_token - params[PRIVATE_TOKEN_PARAM].presence || env[PRIVATE_TOKEN_HEADER].presence - end - # An array of scopes that were registered (using `allow_access_with_scope`) # for the current endpoint class. It also returns scopes registered on # `API::API`, since these are meant to apply to all API routes. diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index eb16701bad5..1490136ee4f 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -7,8 +7,10 @@ module Gitlab attr_reader :request + delegate :params, :env, to: :request + def initialize(request) - @request = ensure_action_dispatch_request(request) + @request = request end def user @@ -16,7 +18,9 @@ module Gitlab end def find_sessionless_user - find_user_from_access_token || find_user_by_rss_token + find_user_from_access_token || find_user_from_rss_token + rescue StandardError + nil end end end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 86f1c13d4b8..dbe2a3a27d1 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -1,16 +1,19 @@ module Gitlab module Auth module UserAuthFinders + PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'.freeze + PRIVATE_TOKEN_PARAM = :private_token + # Check the Rails session for valid authentication details def find_user_from_warden - request.env['warden']&.authenticate if verified_request? + env['warden']&.authenticate if verified_request? end - def find_user_by_rss_token + def find_user_from_rss_token return unless request.format.atom? - token = request.params[:rss_token].presence - return unless token.present? + token = params[:rss_token].presence + return unless token handle_return_value!(User.find_by_rss_token(token)) end @@ -24,14 +27,22 @@ module Gitlab end def validate_access_token!(scopes: []) + return unless access_token + + case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE + raise API::APIGuard::InsufficientScopeError.new(scopes) + when AccessTokenValidationService::EXPIRED + raise API::APIGuard::ExpiredError + when AccessTokenValidationService::REVOKED + raise API::APIGuard::RevokedError + end end private def handle_return_value!(value, &block) - unless value - raise_unauthorized_error? ? raise_unauthorized_error! : return - end + raise API::APIGuard::UnauthorizedError unless value block_given? ? yield(value) : value end @@ -43,13 +54,13 @@ module Gitlab end def private_token - request.params[:private_token].presence || - request.headers['PRIVATE-TOKEN'].presence + params[PRIVATE_TOKEN_PARAM].presence || + env[PRIVATE_TOKEN_HEADER].presence end def find_personal_access_token - token = private_token.to_s - return unless token.present? + token = private_token + return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` handle_return_value!(PersonalAccessToken.find_by(token: token)) @@ -77,18 +88,6 @@ module Gitlab ActionDispatch::Request.new(request.env) end - - def raise_unauthorized_error? - defined?(@raise_unauthorized_error) ? @raise_unauthorized_error : false - end - - def set_raise_unauthorized_error - @raise_unauthorized_error = true - end - - def raise_unauthorized_error! - raise API::APIGuard::UnauthorizedError - end end end end -- cgit v1.2.1 From 21153a4f47871733f3c0d333a10ffa69ada9a5a9 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 9 Nov 2017 19:04:19 +0100 Subject: Homogenising the type of the request handled by UserAuthFinder. Also tests fixed --- lib/api/api_guard.rb | 3 --- lib/gitlab/auth/request_authenticator.rb | 2 -- lib/gitlab/auth/user_auth_finders.rb | 19 +++++++++++-------- 3 files changed, 11 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 01e15ffee84..e2a1a51b300 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,9 +6,6 @@ module API module APIGuard extend ActiveSupport::Concern - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN".freeze - PRIVATE_TOKEN_PARAM = :private_token - included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 1490136ee4f..f500609d1a3 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -7,8 +7,6 @@ module Gitlab attr_reader :request - delegate :params, :env, to: :request - def initialize(request) @request = request end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index dbe2a3a27d1..db900908ead 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -6,13 +6,13 @@ module Gitlab # Check the Rails session for valid authentication details def find_user_from_warden - env['warden']&.authenticate if verified_request? + current_request.env['warden']&.authenticate if verified_request? end def find_user_from_rss_token - return unless request.format.atom? + return unless current_request.format.atom? - token = params[:rss_token].presence + token = current_request.params[:rss_token].presence return unless token handle_return_value!(User.find_by_rss_token(token)) @@ -23,7 +23,7 @@ module Gitlab validate_access_token! - handle_return_value!(access_token&.user) + handle_return_value!(access_token.user) end def validate_access_token!(scopes: []) @@ -54,8 +54,8 @@ module Gitlab end def private_token - params[PRIVATE_TOKEN_PARAM].presence || - env[PRIVATE_TOKEN_HEADER].presence + current_request.params[PRIVATE_TOKEN_PARAM].presence || + current_request.env[PRIVATE_TOKEN_HEADER].presence end def find_personal_access_token @@ -67,7 +67,6 @@ module Gitlab end def find_oauth_access_token - current_request = ensure_action_dispatch_request(request) token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) return unless token @@ -80,7 +79,7 @@ module Gitlab # Check if the request is GET/HEAD, or if CSRF token is valid. def verified_request? - Gitlab::RequestForgeryProtection.verified?(request.env) + Gitlab::RequestForgeryProtection.verified?(current_request.env) end def ensure_action_dispatch_request(request) @@ -88,6 +87,10 @@ module Gitlab ActionDispatch::Request.new(request.env) end + + def current_request + @current_request ||= ensure_action_dispatch_request(request) + end end end end -- cgit v1.2.1 From 2d5397d928cfca222addb250085fe30c72215022 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 10 Nov 2017 11:12:37 +0100 Subject: Removed method handle_return_value --- lib/gitlab/auth/user_auth_finders.rb | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index db900908ead..dc688637107 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -15,7 +15,7 @@ module Gitlab token = current_request.params[:rss_token].presence return unless token - handle_return_value!(User.find_by_rss_token(token)) + User.find_by_rss_token(token) || raise(API::APIGuard::UnauthorizedError) end def find_user_from_access_token @@ -23,7 +23,7 @@ module Gitlab validate_access_token! - handle_return_value!(access_token.user) + access_token.user || raise(API::APIGuard::UnauthorizedError) end def validate_access_token!(scopes: []) @@ -41,12 +41,6 @@ module Gitlab private - def handle_return_value!(value, &block) - raise API::APIGuard::UnauthorizedError unless value - - block_given? ? yield(value) : value - end - def access_token return @access_token if defined?(@access_token) @@ -63,7 +57,7 @@ module Gitlab return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` - handle_return_value!(PersonalAccessToken.find_by(token: token)) + PersonalAccessToken.find_by(token: token) || raise(API::APIGuard::UnauthorizedError) end def find_oauth_access_token @@ -71,10 +65,11 @@ module Gitlab return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` - handle_return_value!(OauthAccessToken.by_token(token)) do |oauth_token| - oauth_token.revoke_previous_refresh_token! - oauth_token - end + oauth_token = OauthAccessToken.by_token(token) + raise(API::APIGuard::UnauthorizedError) unless oauth_token + + oauth_token.revoke_previous_refresh_token! + oauth_token end # Check if the request is GET/HEAD, or if CSRF token is valid. -- cgit v1.2.1 From f1896575237cb92dce5a413bb6b6cc6474cbb19d Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 10 Nov 2017 11:41:33 +0100 Subject: Added some more comments --- lib/api/api_guard.rb | 15 ++++++++------- lib/gitlab/auth/request_authenticator.rb | 2 +- lib/gitlab/auth/user_auth_finders.rb | 10 ++++------ 3 files changed, 13 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index e2a1a51b300..0caf2aa25bc 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -139,13 +139,14 @@ module API # Exceptions # - MissingTokenError = Class.new(StandardError) - TokenNotFoundError = Class.new(StandardError) - ExpiredError = Class.new(StandardError) - RevokedError = Class.new(StandardError) - UnauthorizedError = Class.new(StandardError) - - class InsufficientScopeError < StandardError + AuthenticationException = Class.new(StandardError) + MissingTokenError = Class.new(AuthenticationException) + TokenNotFoundError = Class.new(AuthenticationException) + ExpiredError = Class.new(AuthenticationException) + RevokedError = Class.new(AuthenticationException) + UnauthorizedError = Class.new(AuthenticationException) + + class InsufficientScopeError < AuthenticationException attr_reader :scopes def initialize(scopes) @scopes = scopes.map { |s| s.try(:name) || s } diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index f500609d1a3..8316d0f40d5 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -17,7 +17,7 @@ module Gitlab def find_sessionless_user find_user_from_access_token || find_user_from_rss_token - rescue StandardError + rescue API::APIGuard::AuthenticationException nil end end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index dc688637107..0b4ea3aaf5f 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -47,13 +47,11 @@ module Gitlab @access_token = find_oauth_access_token || find_personal_access_token end - def private_token - current_request.params[PRIVATE_TOKEN_PARAM].presence || + def find_personal_access_token + token = + current_request.params[PRIVATE_TOKEN_PARAM].presence || current_request.env[PRIVATE_TOKEN_HEADER].presence - end - def find_personal_access_token - token = private_token return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` @@ -66,7 +64,7 @@ module Gitlab # Expiration, revocation and scopes are verified in `validate_access_token!` oauth_token = OauthAccessToken.by_token(token) - raise(API::APIGuard::UnauthorizedError) unless oauth_token + raise API::APIGuard::UnauthorizedError unless oauth_token oauth_token.revoke_previous_refresh_token! oauth_token -- cgit v1.2.1 From 29521a313acc03eac0f81058a559fb4ca176f9e7 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 10 Nov 2017 19:17:55 +0100 Subject: Change the rss url guard clause --- lib/gitlab/auth/user_auth_finders.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 0b4ea3aaf5f..b2fb24a4590 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -10,7 +10,7 @@ module Gitlab end def find_user_from_rss_token - return unless current_request.format.atom? + return unless current_request.path.ends_with?('.atom') token = current_request.params[:rss_token].presence return unless token -- cgit v1.2.1 From 98f7982ceccd6f7996774911632943e9f43df6e3 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 10 Nov 2017 20:14:35 +0100 Subject: Leaving atom? query to fix tests --- lib/gitlab/auth/user_auth_finders.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index b2fb24a4590..06b934fa042 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -10,7 +10,7 @@ module Gitlab end def find_user_from_rss_token - return unless current_request.path.ends_with?('.atom') + return unless current_request.path.ends_with?('.atom') || current_request.format.atom? token = current_request.params[:rss_token].presence return unless token -- cgit v1.2.1 From aa84ef1e1af0bac40279e02e4ce889cb660ed9d0 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 16 Nov 2017 15:39:30 +0100 Subject: Moving exceptions to UserAuthFinders --- lib/api/api_guard.rb | 35 ++++++++++------------------------- lib/api/helpers.rb | 2 +- lib/gitlab/auth/user_auth_finders.rb | 32 +++++++++++++++++++++++++------- 3 files changed, 36 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 0caf2aa25bc..a07015406b1 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -93,8 +93,11 @@ module API private def install_error_responders(base) - error_classes = [MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + error_classes = [Gitlab::Auth::UserAuthFinders::MissingTokenError, + Gitlab::Auth::UserAuthFinders::TokenNotFoundError, + Gitlab::Auth::UserAuthFinders::ExpiredError, + Gitlab::Auth::UserAuthFinders::RevokedError, + Gitlab::Auth::UserAuthFinders::InsufficientScopeError] base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend end @@ -103,25 +106,25 @@ module API proc do |e| response = case e - when MissingTokenError + when Gitlab::Auth::UserAuthFinders::MissingTokenError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new - when TokenNotFoundError + when Gitlab::Auth::UserAuthFinders::TokenNotFoundError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Bad Access Token.") - when ExpiredError + when Gitlab::Auth::UserAuthFinders::ExpiredError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Token is expired. You can either do re-authorization or token refresh.") - when RevokedError + when Gitlab::Auth::UserAuthFinders::RevokedError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Token was revoked. You have to re-authorize from the user.") - when InsufficientScopeError + when Gitlab::Auth::UserAuthFinders::InsufficientScopeError # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) # does not include WWW-Authenticate header, which breaks the standard. Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( @@ -134,23 +137,5 @@ module API end end end - - # - # Exceptions - # - - AuthenticationException = Class.new(StandardError) - MissingTokenError = Class.new(AuthenticationException) - TokenNotFoundError = Class.new(AuthenticationException) - ExpiredError = Class.new(AuthenticationException) - RevokedError = Class.new(AuthenticationException) - UnauthorizedError = Class.new(AuthenticationException) - - class InsufficientScopeError < AuthenticationException - attr_reader :scopes - def initialize(scopes) - @scopes = scopes.map { |s| s.try(:name) || s } - end - end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3c8960cb1ab..09e9753b010 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -398,7 +398,7 @@ module API begin @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user! } - rescue APIGuard::UnauthorizedError + rescue Gitlab::Auth::UserAuthFinders::UnauthorizedError unauthorized! end end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 06b934fa042..6ee957a0cd6 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -4,6 +4,24 @@ module Gitlab PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'.freeze PRIVATE_TOKEN_PARAM = :private_token + # + # Exceptions + # + + AuthenticationException = Class.new(StandardError) + MissingTokenError = Class.new(AuthenticationException) + TokenNotFoundError = Class.new(AuthenticationException) + ExpiredError = Class.new(AuthenticationException) + RevokedError = Class.new(AuthenticationException) + UnauthorizedError = Class.new(AuthenticationException) + + class InsufficientScopeError < AuthenticationException + attr_reader :scopes + def initialize(scopes) + @scopes = scopes.map { |s| s.try(:name) || s } + end + end + # Check the Rails session for valid authentication details def find_user_from_warden current_request.env['warden']&.authenticate if verified_request? @@ -15,7 +33,7 @@ module Gitlab token = current_request.params[:rss_token].presence return unless token - User.find_by_rss_token(token) || raise(API::APIGuard::UnauthorizedError) + User.find_by_rss_token(token) || raise(UnauthorizedError) end def find_user_from_access_token @@ -23,7 +41,7 @@ module Gitlab validate_access_token! - access_token.user || raise(API::APIGuard::UnauthorizedError) + access_token.user || raise(UnauthorizedError) end def validate_access_token!(scopes: []) @@ -31,11 +49,11 @@ module Gitlab case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) when AccessTokenValidationService::INSUFFICIENT_SCOPE - raise API::APIGuard::InsufficientScopeError.new(scopes) + raise InsufficientScopeError.new(scopes) when AccessTokenValidationService::EXPIRED - raise API::APIGuard::ExpiredError + raise ExpiredError when AccessTokenValidationService::REVOKED - raise API::APIGuard::RevokedError + raise RevokedError end end @@ -55,7 +73,7 @@ module Gitlab return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` - PersonalAccessToken.find_by(token: token) || raise(API::APIGuard::UnauthorizedError) + PersonalAccessToken.find_by(token: token) || raise(UnauthorizedError) end def find_oauth_access_token @@ -64,7 +82,7 @@ module Gitlab # Expiration, revocation and scopes are verified in `validate_access_token!` oauth_token = OauthAccessToken.by_token(token) - raise API::APIGuard::UnauthorizedError unless oauth_token + raise UnauthorizedError unless oauth_token oauth_token.revoke_previous_refresh_token! oauth_token -- cgit v1.2.1 From 1436598e49792b78f5f753477a9d8c097d666b99 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 16 Nov 2017 17:03:19 +0100 Subject: Moved Exceptions to Gitlab::Auth --- lib/api/api_guard.rb | 20 ++++++++--------- lib/api/helpers.rb | 2 +- lib/gitlab/auth/request_authenticator.rb | 2 +- lib/gitlab/auth/user_auth_finders.rb | 37 ++++++++++++++++---------------- 4 files changed, 31 insertions(+), 30 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index a07015406b1..1953a613f1d 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -93,11 +93,11 @@ module API private def install_error_responders(base) - error_classes = [Gitlab::Auth::UserAuthFinders::MissingTokenError, - Gitlab::Auth::UserAuthFinders::TokenNotFoundError, - Gitlab::Auth::UserAuthFinders::ExpiredError, - Gitlab::Auth::UserAuthFinders::RevokedError, - Gitlab::Auth::UserAuthFinders::InsufficientScopeError] + error_classes = [Gitlab::Auth::MissingTokenError, + Gitlab::Auth::TokenNotFoundError, + Gitlab::Auth::ExpiredError, + Gitlab::Auth::RevokedError, + Gitlab::Auth::InsufficientScopeError] base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend end @@ -106,25 +106,25 @@ module API proc do |e| response = case e - when Gitlab::Auth::UserAuthFinders::MissingTokenError + when Gitlab::Auth::MissingTokenError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new - when Gitlab::Auth::UserAuthFinders::TokenNotFoundError + when Gitlab::Auth::TokenNotFoundError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Bad Access Token.") - when Gitlab::Auth::UserAuthFinders::ExpiredError + when Gitlab::Auth::ExpiredError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Token is expired. You can either do re-authorization or token refresh.") - when Gitlab::Auth::UserAuthFinders::RevokedError + when Gitlab::Auth::RevokedError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Token was revoked. You have to re-authorize from the user.") - when Gitlab::Auth::UserAuthFinders::InsufficientScopeError + when Gitlab::Auth::InsufficientScopeError # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) # does not include WWW-Authenticate header, which breaks the standard. Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 09e9753b010..b26c61ab8da 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -398,7 +398,7 @@ module API begin @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user! } - rescue Gitlab::Auth::UserAuthFinders::UnauthorizedError + rescue Gitlab::Auth::UnauthorizedError unauthorized! end end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 8316d0f40d5..4322fb83cdf 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -17,7 +17,7 @@ module Gitlab def find_sessionless_user find_user_from_access_token || find_user_from_rss_token - rescue API::APIGuard::AuthenticationException + rescue Gitlab::Auth::AuthenticationException nil end end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 6ee957a0cd6..104fff3b56c 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -1,27 +1,28 @@ module Gitlab module Auth + + # + # Exceptions + # + + AuthenticationException = Class.new(StandardError) + MissingTokenError = Class.new(AuthenticationException) + TokenNotFoundError = Class.new(AuthenticationException) + ExpiredError = Class.new(AuthenticationException) + RevokedError = Class.new(AuthenticationException) + UnauthorizedError = Class.new(AuthenticationException) + + class InsufficientScopeError < AuthenticationException + attr_reader :scopes + def initialize(scopes) + @scopes = scopes.map { |s| s.try(:name) || s } + end + end + module UserAuthFinders PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'.freeze PRIVATE_TOKEN_PARAM = :private_token - # - # Exceptions - # - - AuthenticationException = Class.new(StandardError) - MissingTokenError = Class.new(AuthenticationException) - TokenNotFoundError = Class.new(AuthenticationException) - ExpiredError = Class.new(AuthenticationException) - RevokedError = Class.new(AuthenticationException) - UnauthorizedError = Class.new(AuthenticationException) - - class InsufficientScopeError < AuthenticationException - attr_reader :scopes - def initialize(scopes) - @scopes = scopes.map { |s| s.try(:name) || s } - end - end - # Check the Rails session for valid authentication details def find_user_from_warden current_request.env['warden']&.authenticate if verified_request? -- cgit v1.2.1 From b810f479d55b535b31a723975926762b5ef42cbe Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 16 Nov 2017 17:52:54 +0100 Subject: Removing Offender --- lib/gitlab/auth/user_auth_finders.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 104fff3b56c..15b54f176b9 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -1,6 +1,5 @@ module Gitlab module Auth - # # Exceptions # -- cgit v1.2.1 From 7f0317917a6684189b1637ea73f90d258e8a72b6 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 17 Nov 2017 10:09:56 +0100 Subject: Changes after rebase --- lib/api/api_guard.rb | 21 +-------------------- lib/gitlab/auth/user_auth_finders.rb | 8 +++++--- 2 files changed, 6 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 1953a613f1d..9aeebc34525 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -39,7 +39,7 @@ module API # Helper Methods for Grape Endpoint module HelperMethods - include Gitlab::Utils::StrongMemoize + include Gitlab::Auth::UserAuthFinders def find_current_user! user = find_user_from_access_token || find_user_from_warden @@ -50,25 +50,6 @@ module API user end - def access_token - strong_memoize(:access_token) do - find_oauth_access_token || find_personal_access_token - end - end - - def validate_access_token!(scopes: []) - return unless access_token - - case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) - when AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when AccessTokenValidationService::EXPIRED - raise ExpiredError - when AccessTokenValidationService::REVOKED - raise RevokedError - end - end - private # An array of scopes that were registered (using `allow_access_with_scope`) diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 15b54f176b9..cd497fe1cdb 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -19,6 +19,8 @@ module Gitlab end module UserAuthFinders + include Gitlab::Utils::StrongMemoize + PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'.freeze PRIVATE_TOKEN_PARAM = :private_token @@ -60,9 +62,9 @@ module Gitlab private def access_token - return @access_token if defined?(@access_token) - - @access_token = find_oauth_access_token || find_personal_access_token + strong_memoize(:access_token) do + find_oauth_access_token || find_personal_access_token + end end def find_personal_access_token -- cgit v1.2.1 From 5cecff893db3188ff5ec779e07e9c598d3ed2e21 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 16 Nov 2017 16:14:24 -0800 Subject: Convert migration to populate latest merge request ID into a background migration This is to smear updates over a few hours to avoid causing excessive replication lag as seen in https://gitlab.com/gitlab-com/infrastructure/issues/3235. --- ..._merge_requests_latest_merge_request_diff_id.rb | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb new file mode 100644 index 00000000000..7e109e96e73 --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb @@ -0,0 +1,30 @@ +module Gitlab + module BackgroundMigration + class PopulateMergeRequestsLatestMergeRequestDiffId + BATCH_SIZE = 1_000 + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include ::EachBatch + end + + def perform(start_id, stop_id) + update = ' + latest_merge_request_diff_id = ( + SELECT MAX(id) + FROM merge_request_diffs + WHERE merge_requests.id = merge_request_diffs.merge_request_id + )'.squish + + MergeRequest + .where(id: start_id..stop_id) + .where(latest_merge_request_diff_id: nil) + .each_batch(of: BATCH_SIZE) do |relation| + + relation.update_all(update) + end + end + end + end +end -- cgit v1.2.1 From f767dd4a4da2086de699eb55f468a646846b4632 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 17 Nov 2017 12:17:16 +0100 Subject: Fix go-import meta data when enabled_git_access_protocol is a blank string --- lib/gitlab/middleware/go.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index cfc6b2a2029..1edda50e4b0 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -42,12 +42,11 @@ module Gitlab project_url = URI.join(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - repository_url = case current_application_settings.enabled_git_access_protocol - when 'ssh' + repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh' shell = config.gitlab_shell port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" - when 'http', nil + else "#{project_url}.git" end -- cgit v1.2.1 From ff26ea818cb92eabc8fbea1a8385cb5bc6b0a66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Fri, 17 Nov 2017 11:48:32 +0000 Subject: Resolve "Performance issues when loading large number of wiki pages" --- lib/gitlab/git/wiki.rb | 56 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 022d1f249a9..d4a53d32c28 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -58,12 +58,12 @@ module Gitlab end end - def pages - @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| + def pages(limit: nil) + @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_get_all_pages else - gollum_get_all_pages + gollum_get_all_pages(limit: limit) end end end @@ -88,14 +88,23 @@ module Gitlab end end - def page_versions(page_path) + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def page_versions(page_path, options = {}) current_page = gollum_page_by_path(page_path) - current_page.versions.map do |gollum_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) - new_version(gollum_page, gollum_git_commit.id) + + commits_from_page(current_page, options).map do |gitlab_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) + Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) end end + def count_page_versions(page_path) + @repository.count_commits(ref: 'HEAD', path: page_path) + end + def preview_slug(title, format) # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid # using Rugged through a Gollum::Wiki instance @@ -110,6 +119,22 @@ module Gitlab private + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def commits_from_page(gollum_page, options = {}) + unless options[:limit] + options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page + options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i + end + + @repository.log(ref: gollum_page.last_version.id, + path: gollum_page.path, + limit: options[:limit], + offset: options[:offset]) + end + def gollum_wiki @gollum_wiki ||= Gollum::Wiki.new(@repository.path) end @@ -126,8 +151,17 @@ module Gitlab end def new_version(gollum_page, commit_id) - commit = Gitlab::Git::Commit.find(@repository, commit_id) - Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format) + end + + def version(commit_id) + commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) } + + if RequestStore.active? + RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } + else + commit_find_proc.call + end end def assert_type!(object, klass) @@ -185,8 +219,8 @@ module Gitlab Gitlab::Git::WikiFile.new(gollum_file) end - def gollum_get_all_pages - gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + def gollum_get_all_pages(limit: nil) + gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) } end def gitaly_write_page(name, format, content, commit_details) -- cgit v1.2.1 From 4188c10c07d7b9bfaee5046ecfcc88cf8cca456b Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 17 Nov 2017 13:33:21 +0100 Subject: Renaming AuthenticationException to AuthenticationError --- lib/gitlab/auth/request_authenticator.rb | 2 +- lib/gitlab/auth/user_auth_finders.rb | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 4322fb83cdf..46ec040ce92 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -17,7 +17,7 @@ module Gitlab def find_sessionless_user find_user_from_access_token || find_user_from_rss_token - rescue Gitlab::Auth::AuthenticationException + rescue Gitlab::Auth::AuthenticationError nil end end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index cd497fe1cdb..b4114a3ac96 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -4,14 +4,14 @@ module Gitlab # Exceptions # - AuthenticationException = Class.new(StandardError) - MissingTokenError = Class.new(AuthenticationException) - TokenNotFoundError = Class.new(AuthenticationException) - ExpiredError = Class.new(AuthenticationException) - RevokedError = Class.new(AuthenticationException) - UnauthorizedError = Class.new(AuthenticationException) - - class InsufficientScopeError < AuthenticationException + AuthenticationError = Class.new(StandardError) + MissingTokenError = Class.new(AuthenticationError) + TokenNotFoundError = Class.new(AuthenticationError) + ExpiredError = Class.new(AuthenticationError) + RevokedError = Class.new(AuthenticationError) + UnauthorizedError = Class.new(AuthenticationError) + + class InsufficientScopeError < AuthenticationError attr_reader :scopes def initialize(scopes) @scopes = scopes.map { |s| s.try(:name) || s } -- cgit v1.2.1 From c7cf68bd6ff744e044944acad586e06badc481d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Fri, 17 Nov 2017 14:24:25 +0000 Subject: Changing OAuth lookup to be case insensitive --- lib/gitlab/ldap/user.rb | 5 +---- lib/gitlab/o_auth/user.rb | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 4d5c67ed892..3945df27eed 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -9,11 +9,8 @@ module Gitlab class User < Gitlab::OAuth::User class << self def find_by_uid_and_provider(uid, provider) - uid = Gitlab::LDAP::Person.normalize_dn(uid) + identity = ::Identity.with_extern_uid(provider, uid).take - identity = ::Identity - .where(provider: provider) - .where(extern_uid: uid).last identity && identity.user end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index b4b3b00c84d..552133234a3 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -157,7 +157,7 @@ module Gitlab end def find_by_uid_and_provider - identity = Identity.find_by(provider: auth_hash.provider, extern_uid: auth_hash.uid) + identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take identity && identity.user end -- cgit v1.2.1 From 5a335c4d955bbb74a50ca30ac45d30f6ef66774d Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 17 Nov 2017 15:55:43 +0100 Subject: Remove the selects when counting the last page The last page of the first collection is only loaded into memory when it is being viewed. If it isn't loaded into memory, the `#size` call triggers a count query. This `#count` would generate an invalid query if our custom preloaded counts are included by adding a separate `as count_column` alias on top of the count aliases. Removing the selects in this case will make sure a valid `COUNT(*)` is generated. --- lib/gitlab/multi_collection_paginator.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index eb3c9002710..c22d0a84860 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -55,7 +55,9 @@ module Gitlab def first_collection_last_page_size return @first_collection_last_page_size if defined?(@first_collection_last_page_size) - @first_collection_last_page_size = paginated_first_collection(first_collection_page_count).count + @first_collection_last_page_size = paginated_first_collection(first_collection_page_count) + .except(:select) + .size end end end -- cgit v1.2.1 From 4d367dd40037500fd7b96059fa300cbe2ce28e85 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 17 Nov 2017 16:02:10 +0000 Subject: Add computed update docs for update_column_in_batches --- lib/gitlab/database/migration_helpers.rb | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 2c35da8f1aa..c276c3566b4 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -220,6 +220,15 @@ module Gitlab # column - The name of the column to update. # value - The value for the column. # + # The `value` argument is typically a literal. To perform a computed + # update, an Arel literal can be used instead: + # + # update_value = Arel.sql('bar * baz') + # + # update_column_in_batches(:projects, :foo, update_value) do |table, query| + # query.where(table[:some_column].eq('hello')) + # end + # # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop # determines this method to be too complex while there's no way to make it # less "complex" without introducing extra methods (which actually will -- cgit v1.2.1 From 64a9e53bd16092e869f88e42a3e69f3f4ba0a23e Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 17 Nov 2017 17:57:48 +0000 Subject: Fix conflict highlighting Conflicts used to take a `Repository` and pass that to `Gitlab::Highlight.highlight`, which would call `#gitattribute` on the repository. Now they use a `Gitlab::Git::Repository`, which didn't have that method defined - but defining it on `Gitlab::Git::Repository` does make it available on `Repository` through `method_missing`, so we can do that and both cases will work. --- lib/gitlab/git/repository.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index cfb88a0c12b..b97d07187ae 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -984,6 +984,10 @@ module Gitlab @attributes.attributes(path) end + def gitattribute(path, name) + attributes(path)[name] + end + def languages(ref = nil) Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| if is_enabled -- cgit v1.2.1 From 38730a2d07c1b956ec578d090b3019a574ac5cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Fri, 17 Nov 2017 13:54:48 -0300 Subject: Incorporate Gitaly's RefService.DeleteRefs RPC --- lib/gitlab/git/repository.rb | 8 +++++++- lib/gitlab/gitaly_client/ref_service.rb | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index cfb88a0c12b..798969932e3 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -304,7 +304,13 @@ module Gitlab end def delete_all_refs_except(prefixes) - delete_refs(*all_ref_names_except(prefixes)) + gitaly_migrate(:ref_delete_refs) do |is_enabled| + if is_enabled + gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) + else + delete_refs(*all_ref_names_except(prefixes)) + end + end end # Returns an Array of all ref names, except when it's matching pattern diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index b0c73395cb1..334dfd0f2e8 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -126,6 +126,15 @@ module Gitlab GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) end + def delete_refs(except_with_prefixes:) + request = Gitaly::DeleteRefsRequest.new( + repository: @gitaly_repo, + except_with_prefix: except_with_prefixes + ) + + GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + end + private def consume_refs_response(response) -- cgit v1.2.1 From 3f0c9e97088c27093cc0f09744261366f9d1c352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Mon, 20 Nov 2017 09:08:09 +0000 Subject: Fix Gitlab::Git::Repository#remote_tags using unexisting variable --- lib/gitlab/git/repository_mirroring.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index 637e7a0659c..4500482d68f 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -78,7 +78,7 @@ module Gitlab def list_remote_tags(remote) tag_list, exit_code, error = nil - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote}) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote}) Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| tag_list = stdout.read @@ -88,7 +88,7 @@ module Gitlab raise RemoteError, error unless exit_code.zero? - tag_list.split('\n') + tag_list.split("\n") end end end -- cgit v1.2.1 From 936e9e895000c9f9ae39713c969b325f6b59c0c3 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 14 Nov 2017 16:58:56 +0100 Subject: Clean up schema of the "merge_requests" table This adds various foreign keys and indexes to the "merge_requests" table as outlined in https://gitlab.com/gitlab-org/gitlab-ce/issues/31825. Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/31825 --- lib/gitlab/import_export/merge_request_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 61db4bd9ccc..f3d7407383c 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -1,7 +1,7 @@ module Gitlab module ImportExport class MergeRequestParser - FORKED_PROJECT_ID = -1 + FORKED_PROJECT_ID = nil def initialize(project, diff_head_sha, merge_request, relation_hash) @project = project -- cgit v1.2.1 From fa39e8a09c310dcbb5ce04a61de754e4fdb6c51d Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 20 Nov 2017 10:39:56 +0100 Subject: Don't move project repository/attachments when using hashed storage When a project is using hashed storage, the repositories and attachments wouldn't be saved on disk using the `full_path`. So the migration would not do anything. However: best to just skip moving when hashed storage is enabled. --- .../rename_reserved_paths_migration/v1/migration_classes.rb | 12 ++++++++++++ .../rename_reserved_paths_migration/v1/rename_projects.rb | 8 +++++--- 2 files changed, 17 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index 5481024db8e..7e492938eac 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -68,6 +68,11 @@ module Gitlab has_one :route, as: :source self.table_name = 'projects' + HASHED_STORAGE_FEATURES = { + repository: 1, + attachments: 2 + }.freeze + def repository_storage_path Gitlab.config.repositories.storages[repository_storage]['path'] end @@ -76,6 +81,13 @@ module Gitlab def self.name 'Project' end + + def hashed_storage?(feature) + raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature) + return false unless respond_to?(:storage_version) + + self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature] + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 75a75f61953..d32616862f0 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -22,9 +22,11 @@ module Gitlab end def move_project_folders(project, old_full_path, new_full_path) - move_repository(project, old_full_path, new_full_path) - move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") - move_uploads(old_full_path, new_full_path) + unless project.hashed_storage?(:repository) + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + end + move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments) move_pages(old_full_path, new_full_path) end -- cgit v1.2.1 From c82ae26565fdbffd205476929a2c08c9448a1365 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Mon, 20 Nov 2017 16:09:56 +0000 Subject: Clarify wording of protected branch settings for the default branch No-one is allowed to force push to a protected branch, or delete it. That's correct in the documentation, but was wrong in the drop-down. --- lib/gitlab/access.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index b4012ebbb99..7127948cf00 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -58,9 +58,9 @@ module Gitlab def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, - "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, - "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL + "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Masters can push to the branch." => PROTECTION_DEV_CAN_MERGE, + "Partially protected: Both developers and masters can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH, + "Fully protected: Developers cannot push new commits, but masters can. No-one can force push or delete the branch." => PROTECTION_FULL } end -- cgit v1.2.1 From f2977c63a8e61ccc5d38b083851f0f8769691036 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Mon, 20 Nov 2017 15:16:29 +0000 Subject: Fix bitbucket wiki import with hashed storage enabled --- lib/gitlab/bitbucket_import/importer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 033ecd15749..d48ae17aeaf 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -61,9 +61,9 @@ module Gitlab def import_wiki return if project.wiki.repository_exists? - path_with_namespace = "#{project.full_path}.wiki" + disk_path = project.wiki.disk_path import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) + gitlab_shell.import_repository(project.repository_storage_path, disk_path, import_url) rescue StandardError => e errors << { type: :wiki, errors: e.message } end -- cgit v1.2.1 From d8dee231bff1058f29f38c70a5bb549fdf62438b Mon Sep 17 00:00:00 2001 From: digitalMoksha Date: Mon, 20 Nov 2017 17:23:12 +0100 Subject: remove the rake task `gitlab:sidekiq:drop_post_receive` This task is no longer being used and is not documented. --- lib/tasks/gitlab/sidekiq.rake | 47 ------------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 lib/tasks/gitlab/sidekiq.rake (limited to 'lib') diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake deleted file mode 100644 index 6cbc83b8973..00000000000 --- a/lib/tasks/gitlab/sidekiq.rake +++ /dev/null @@ -1,47 +0,0 @@ -namespace :gitlab do - namespace :sidekiq do - QUEUE = 'queue:post_receive'.freeze - - desc 'Drop all Sidekiq PostReceive jobs for a given project' - task :drop_post_receive, [:project] => :environment do |t, args| - unless args.project.present? - abort "Please specify the project you want to drop PostReceive jobs for:\n rake gitlab:sidekiq:drop_post_receive[group/project]" - end - project_path = Project.find_by_full_path(args.project).repository.path_to_repo - - Sidekiq.redis do |redis| - unless redis.exists(QUEUE) - abort "Queue #{QUEUE} is empty" - end - - temp_queue = "#{QUEUE}_#{Time.now.to_i}" - redis.rename(QUEUE, temp_queue) - - # At this point, then post_receive queue is empty. It may be receiving - # new jobs already. We will repopulate it with the old jobs, skipping the - # ones we want to drop. - dropped = 0 - while (job = redis.lpop(temp_queue)) - if repo_path(job) == project_path - dropped += 1 - else - redis.rpush(QUEUE, job) - end - end - # The temp_queue will delete itself after we have popped all elements - # from it - - puts "Dropped #{dropped} jobs containing #{project_path} from #{QUEUE}" - end - end - - def repo_path(job) - job_args = JSON.parse(job)['args'] - if job_args - job_args.first - else - nil - end - end - end -end -- cgit v1.2.1 From 3c52e2f06ef3234ab5ace532e21e194abab96b59 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 20 Nov 2017 15:27:52 -0800 Subject: Optimize read-only middleware so that it does not consume as much CPU In !15082, we changed the behavior of the middleware to call `Rails.application.routes.recognize_path` whenever a new route arrived. However, this can be a CPU-intensive task because Rails needs to allocate memory and compile 850+ different regular expressions, which are complicated in GitLab. As a short-term fix, we can do a lightweight string match before we do the heavier comparison. Closes #40185, gitlab-com/infrastructure#3240 --- lib/gitlab/middleware/read_only.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 5e4932e4e57..dc77f737da8 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -74,10 +74,16 @@ module Gitlab end def grack_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('.git/git-upload-pack') + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' end def lfs_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('/info/lfs/objects/batch') + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' end end -- cgit v1.2.1 From f80654c4e13ff42ca03bacb59b924b9960f59148 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 20 Nov 2017 23:28:49 -0800 Subject: Memoize GitlabShellAdapter for performance and ease of testing Port of https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3463#note_47990536 --- lib/gitlab/shell_adapter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/shell_adapter.rb b/lib/gitlab/shell_adapter.rb index fbe2a7a0d72..053dd4ab9e0 100644 --- a/lib/gitlab/shell_adapter.rb +++ b/lib/gitlab/shell_adapter.rb @@ -5,7 +5,7 @@ module Gitlab module ShellAdapter def gitlab_shell - Gitlab::Shell.new + @gitlab_shell ||= Gitlab::Shell.new end end end -- cgit v1.2.1 From 6f1e9f7ef773ead892a1d43ec15a38dfb486e229 Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Tue, 21 Nov 2017 12:28:02 +0000 Subject: Fix slow gitaly dev test bundle --- lib/tasks/gitlab/gitaly.rake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 8377fe3269d..87835dbe719 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -17,9 +17,7 @@ namespace :gitlab do _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? ['gmake'] : ['make'] - if Rails.env.test? - command += %W[BUNDLE_PATH=#{Bundler.bundle_path}] - end + command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? Dir.chdir(args.dir) do create_gitaly_configuration -- cgit v1.2.1 From cba68d338bfac97baab153d38fa63b85a27f6d19 Mon Sep 17 00:00:00 2001 From: digitalMoksha Date: Tue, 21 Nov 2017 13:29:57 +0100 Subject: use `Gitlab::Routing.url_helpers` instead of `Rails.application.routes.url_helpers` since `Rails.application.routes.url_helpers` creates a new anonymous module every time it's called --- lib/gitlab/middleware/read_only.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index dc77f737da8..c26656704d7 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -58,7 +58,7 @@ module Gitlab end def last_visited_url - @env['HTTP_REFERER'] || rack_session['user_return_to'] || Rails.application.routes.url_helpers.root_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url end def route_hash -- cgit v1.2.1 From 91075c8237307e09c2be8a88ffb3711fd62417d1 Mon Sep 17 00:00:00 2001 From: digitalMoksha Date: Tue, 21 Nov 2017 13:30:54 +0100 Subject: check for `read_only?` first before seeing if request is disallowed --- lib/gitlab/middleware/read_only.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index c26656704d7..f8b9c5bff4c 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -14,7 +14,7 @@ module Gitlab @env = env @route_hash = nil - if disallowed_request? && Gitlab::Database.read_only? + if Gitlab::Database.read_only? && disallowed_request? Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') error_message = 'You cannot do writing operations on a read-only GitLab instance' -- cgit v1.2.1 From f9565e303916ca194ef63b5fd3de541bf1c2a170 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Fri, 3 Nov 2017 14:16:43 +0100 Subject: Batchload blobs for diff generation After installing a new gem, batch-loader, a construct can be used to queue data to be fetched in bulk. The gem was also introduced in both gitlab-org/gitlab-ce!14680 and gitlab-org/gitlab-ce!14846, but those mrs are not merged yet. For the generation of diffs, both the old blob and the new blob need to be loaded. This for every file in the diff, too. Now we collect all these so we do 1 fetch. Three `.allow_n_plus_1_calls` have been removed, which I expect to be valid, but this needs to be confirmed by a full CI run. Possibly closes: - https://gitlab.com/gitlab-org/gitlab-ce/issues/37445 - https://gitlab.com/gitlab-org/gitlab-ce/issues/37599 - https://gitlab.com/gitlab-org/gitlab-ce/issues/37431 --- lib/gitlab/diff/file.rb | 18 ++++++++---------- lib/gitlab/diff/file_collection/base.rb | 5 +---- lib/gitlab/git/blob.rb | 2 ++ lib/gitlab/git/repository.rb | 5 +++++ 4 files changed, 16 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index ea5891a028a..d0cfe2386ca 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -25,6 +25,10 @@ module Gitlab @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + + # Ensure items are collected in the the batch + new_blob + old_blob end def position(position_marker, position_type: :text) @@ -95,21 +99,15 @@ module Gitlab end def new_blob - return @new_blob if defined?(@new_blob) - - sha = new_content_sha - return @new_blob = nil unless sha + return unless new_content_sha - @new_blob = repository.blob_at(sha, file_path) + Blob.lazy(repository.project, new_content_sha, file_path) end def old_blob - return @old_blob if defined?(@old_blob) - - sha = old_content_sha - return @old_blob = nil unless sha + return unless old_content_sha - @old_blob = repository.blob_at(sha, old_path) + Blob.lazy(repository.project, old_content_sha, old_path) end def content_sha diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 88ae65cb468..a6007ebf531 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -22,10 +22,7 @@ module Gitlab end def diff_files - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37445 - Gitlab::GitalyClient.allow_n_plus_1_calls do - @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } - end + @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } end def diff_file_with_old_path(old_path) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index bd5039fb87e..ddd52136bc4 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -179,6 +179,8 @@ module Gitlab ) end end + rescue Rugged::ReferenceError + nil end def rugged_raw(repository, sha, limit:) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 0c522deb6fa..3cb9b254e6e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1161,6 +1161,11 @@ module Gitlab Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) end + # Items should be of format [[commit_id, path], [commit_id1, path1]] + def batch_blobs(items, blob_size_limit: nil) + Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) + end + def commit_index(user, branch_name, index, options) committer = user_to_committer(user) -- cgit v1.2.1 From c900c21eef9235306d7d0da42b07aa2de346e263 Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Tue, 21 Nov 2017 08:31:23 -0500 Subject: add `#with_metadata` scope to remove a N+1 from the notes' API --- lib/api/notes.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 0b9ab4eeb05..ceaaeca4046 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -33,7 +33,7 @@ module API # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(noteable.notes) + paginate(noteable.notes.with_metadata) .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note else @@ -50,7 +50,7 @@ module API end get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do noteable = find_project_noteable(noteables_str, params[:noteable_id]) - note = noteable.notes.find(params[:note_id]) + note = noteable.notes.with_metadata.find(params[:note_id]) can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) if can_read_note -- cgit v1.2.1 From aeb2f49fd4c65b9f5e3b2d7858a250a747a0f702 Mon Sep 17 00:00:00 2001 From: digitalMoksha Date: Tue, 21 Nov 2017 15:35:30 +0100 Subject: Revert "check for `read_only?` first before seeing if request is disallowed" This reverts commit 91075c8237307e09c2be8a88ffb3711fd62417d1. --- lib/gitlab/middleware/read_only.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index f8b9c5bff4c..c26656704d7 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -14,7 +14,7 @@ module Gitlab @env = env @route_hash = nil - if Gitlab::Database.read_only? && disallowed_request? + if disallowed_request? && Gitlab::Database.read_only? Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') error_message = 'You cannot do writing operations on a read-only GitLab instance' -- cgit v1.2.1 From 0b9e1e16626eff4cd8ae43ce47ec0f965beaf843 Mon Sep 17 00:00:00 2001 From: Daniel Juarez Date: Tue, 21 Nov 2017 15:47:58 +0000 Subject: Skip confirmation user api --- lib/api/users.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index d80b364bd09..0cd89b1bcf8 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -31,7 +31,6 @@ module API optional :location, type: String, desc: 'The location of the user' optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' - optional :skip_confirmation, type: Boolean, default: false, desc: 'Flag indicating the account is confirmed' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' optional :avatar, type: File, desc: 'Avatar image for user' all_or_none_of :extern_uid, :provider @@ -101,6 +100,7 @@ module API requires :email, type: String, desc: 'The email of the user' optional :password, type: String, desc: 'The password of the new user' optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token' + optional :skip_confirmation, type: Boolean, desc: 'Flag indicating the account is confirmed' at_least_one_of :password, :reset_password requires :name, type: String, desc: 'The name of the user' requires :username, type: String, desc: 'The username of the user' @@ -134,6 +134,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' optional :email, type: String, desc: 'The email of the user' optional :password, type: String, desc: 'The password of the new user' + optional :skip_reconfirmation, type: Boolean, desc: 'Flag indicating the account skips the confirmation by email' optional :name, type: String, desc: 'The name of the user' optional :username, type: String, desc: 'The username of the user' use :optional_attributes -- cgit v1.2.1 From afd5911557051c4dae652b386c7a91f746ff18a5 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 21 Nov 2017 17:13:21 +0000 Subject: Set the default gitlab-shell timeout to 3 hours --- lib/gitlab/shell.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index dc0184e4ad9..996d213fdb4 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -101,8 +101,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) - # Timeout should be less than 900 ideally, to prevent the memory killer - # to silently kill the process without knowing we are timing out here. + # The timeout ensures the subprocess won't hang forever cmd = [gitlab_shell_projects_path, 'import-project', storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"] gitlab_shell_fast_execute_raise_error(cmd) -- cgit v1.2.1 From c33ca9d807a70be51afea2692b7191d7790edd6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Tue, 21 Nov 2017 17:45:36 +0000 Subject: Use `make install` for Gitaly setups in non-test environments --- lib/tasks/gitlab/gitaly.rake | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 87835dbe719..f2002d7a426 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -14,8 +14,10 @@ namespace :gitlab do checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) + command = %w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + _, status = Gitlab::Popen.popen(%w[which gmake]) - command = status.zero? ? ['gmake'] : ['make'] + command << (status.zero? ? 'gmake' : 'make') command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? @@ -23,7 +25,7 @@ namespace :gitlab do create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? - Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + command) } + Bundler.with_original_env { run_command!(command) } end end end @@ -80,9 +82,12 @@ namespace :gitlab do end def create_gitaly_configuration - File.open("config.toml", "w") do |f| + File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f| f.puts gitaly_configuration_toml end + rescue Errno::EEXIST + puts "Skipping config.toml generation:" + puts "A configuration file already exists." rescue ArgumentError => e puts "Skipping config.toml generation:" puts e.message -- cgit v1.2.1 From 5e861a052551d307e928fa0797a64d30da380eeb Mon Sep 17 00:00:00 2001 From: Brett Walker Date: Tue, 21 Nov 2017 20:26:53 +0000 Subject: ignore hashed repos (for now) when using `rake gitlab:cleanup:repos` --- lib/tasks/gitlab/cleanup.rake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 91c74bfb6b4..301affc9522 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -59,7 +59,10 @@ namespace :gitlab do .sub(%r{^/*}, '') .chomp('.git') .chomp('.wiki') - next if Project.find_by_full_path(repo_with_namespace) + + # TODO ignoring hashed repositories for now. But revisit to fully support + # possible orphaned hashed repos + next if repo_with_namespace.start_with?('@hashed/') || Project.find_by_full_path(repo_with_namespace) new_path = path + move_suffix puts path.inspect + ' -> ' + new_path.inspect -- cgit v1.2.1 From 131e74d10dafbf2b781ab5d5517e42a18e20a587 Mon Sep 17 00:00:00 2001 From: "Vitaliy @blackst0ne Klachkov" Date: Wed, 22 Nov 2017 14:12:04 +1100 Subject: Add support of Mermaid --- lib/banzai/filter/mermaid_filter.rb | 20 ++++++++++++++++ lib/banzai/filter/syntax_highlight_filter.rb | 35 +++++++++++++++++----------- lib/banzai/pipeline/gfm_pipeline.rb | 1 + lib/rouge/lexers/math.rb | 9 ------- lib/rouge/lexers/plantuml.rb | 9 ------- 5 files changed, 42 insertions(+), 32 deletions(-) create mode 100644 lib/banzai/filter/mermaid_filter.rb delete mode 100644 lib/rouge/lexers/math.rb delete mode 100644 lib/rouge/lexers/plantuml.rb (limited to 'lib') diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb new file mode 100644 index 00000000000..b545b947a2c --- /dev/null +++ b/lib/banzai/filter/mermaid_filter.rb @@ -0,0 +1,20 @@ +module Banzai + module Filter + class MermaidFilter < HTML::Pipeline::Filter + def call + doc.css('pre[lang="mermaid"]').add_class('mermaid') + doc.css('pre[lang="mermaid"]').add_class('js-render-mermaid') + + # The `` blocks are added in the lib/banzai/filter/syntax_highlight_filter.rb + # We want to keep context and consistency, so we the blocks are added for all filters. + # Details: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15107/diffs?diff_id=7962900#note_45495859 + doc.css('pre[lang="mermaid"]').each do |pre| + document = pre.at('code') + document.replace(document.content) + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 7da565043d1..a79a0154846 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,23 +14,26 @@ module Banzai end def highlight_node(node) - language = node.attr('lang') code = node.text - css_classes = "code highlight" - lexer = lexer_for(language) - lang = lexer.tag - - begin - code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: lang) - - css_classes << " js-syntax-highlight #{lang}" - rescue - lang = nil - # Gracefully handle syntax highlighter bugs/errors to ensure - # users can still access an issue/comment/etc. + css_classes = 'code highlight js-syntax-highlight' + language = node.attr('lang') + + if use_rouge?(language) + lexer = lexer_for(language) + language = lexer.tag + + begin + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: language) + css_classes << " #{language}" + rescue + # Gracefully handle syntax highlighter bugs/errors to ensure + # users can still access an issue/comment/etc. + + language = nil + end end - highlighted = %(
#{code}
) + highlighted = %(
#{code}
) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) @@ -51,6 +54,10 @@ module Banzai # Replace the parent `pre` element with the entire highlighted block node.parent.replace(highlighted) end + + def use_rouge?(language) + %w(math mermaid plantuml).exclude?(language) + end end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 3208abfc538..55874ad50a3 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -14,6 +14,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::MathFilter, + Filter::MermaidFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb deleted file mode 100644 index 939b23a3421..00000000000 --- a/lib/rouge/lexers/math.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Rouge - module Lexers - class Math < PlainText - title "A passthrough lexer used for LaTeX input" - desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter" - tag 'math' - end - end -end diff --git a/lib/rouge/lexers/plantuml.rb b/lib/rouge/lexers/plantuml.rb deleted file mode 100644 index 63c461764fc..00000000000 --- a/lib/rouge/lexers/plantuml.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Rouge - module Lexers - class Plantuml < PlainText - title "A passthrough lexer used for PlantUML input" - desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter" - tag 'plantuml' - end - end -end -- cgit v1.2.1 From 571f1dda17203d162dac65270a6b68366de1f101 Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Wed, 22 Nov 2017 10:19:42 +0000 Subject: Add FetchSourceBranch Gitaly call --- lib/gitlab/git/repository.rb | 20 +++++++++++++++----- lib/gitlab/gitaly_client.rb | 12 +++++++++--- lib/gitlab/gitaly_client/repository_service.rb | 19 +++++++++++++++++++ lib/tasks/gitlab/gitaly.rake | 2 ++ 4 files changed, 45 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3cb9b254e6e..dcca20c75ef 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1058,12 +1058,11 @@ module Gitlab end def fetch_source_branch!(source_repository, source_branch, local_ref) - with_repo_branch_commit(source_repository, source_branch) do |commit| - if commit - write_ref(local_ref, commit.sha) - true + Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled| + if is_enabled + gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) else - false + rugged_fetch_source_branch(source_repository, source_branch, local_ref) end end end @@ -1216,6 +1215,17 @@ module Gitlab private + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) + with_repo_branch_commit(source_repository, source_branch) do |commit| + if commit + write_ref(local_ref, commit.sha) + true + else + false + end + end + end + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 0b35a787e07..572f4c892f6 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -75,6 +75,10 @@ module Gitlab address end + def self.address_metadata(storage) + Base64.strict_encode64(JSON.dump({ storage => { 'address' => address(storage), 'token' => token(storage) } })) + end + # All Gitaly RPC call sites should use GitalyClient.call. This method # makes sure that per-request authentication headers are set. # @@ -89,18 +93,19 @@ module Gitlab # kwargs.merge(deadline: Time.now + 10) # end # - def self.call(storage, service, rpc, request) + def self.call(storage, service, rpc, request, remote_storage: nil) start = Process.clock_gettime(Process::CLOCK_MONOTONIC) enforce_gitaly_request_limits(:call) - kwargs = request_kwargs(storage) + kwargs = request_kwargs(storage, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? + stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend ensure self.query_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start end - def self.request_kwargs(storage) + def self.request_kwargs(storage, remote_storage: nil) encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { 'authorization' => "Bearer #{encoded_token}", @@ -110,6 +115,7 @@ module Gitlab feature_stack = Thread.current[:gitaly_feature_stack] feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature + metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage { metadata: metadata } end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index cef692d3c2a..70cb16bd810 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -65,6 +65,25 @@ module Gitlab response.value end + + def fetch_source_branch(source_repository, source_branch, local_ref) + request = Gitaly::FetchSourceBranchRequest.new( + repository: @gitaly_repo, + source_repository: source_repository.gitaly_repository, + source_branch: source_branch.b, + target_ref: local_ref.b + ) + + response = GitalyClient.call( + @storage, + :repository_service, + :fetch_source_branch, + request, + remote_storage: source_repository.storage + ) + + response.result + end end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index f2002d7a426..4d880c05f99 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -78,6 +78,8 @@ namespace :gitlab do config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } + config[:bin_dir] = Gitlab.config.gitaly.client_path + TOML.dump(config) end -- cgit v1.2.1 From 05e0c4c1c57077735dd91564750696dc76b1b90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 20 Nov 2017 17:16:45 +0100 Subject: Try to find the merge-base against the canonical master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also, name the remotes in Gitlab::EeCompatCheck Signed-off-by: Rémy Coutable --- lib/gitlab/ee_compat_check.rb | 69 ++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index efc2e46d289..3a6165f504c 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -31,16 +31,22 @@ module Gitlab def check ensure_patches_dir - generate_patch(ce_branch, ce_patch_full_path) + add_remote('canonical-ce', "#{DEFAULT_CE_PROJECT_URL}.git") + generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, remote: 'canonical-ce') ensure_ee_repo Dir.chdir(ee_repo_dir) do step("In the #{ee_repo_dir} directory") + add_remote('canonical-ee', EE_REPO_URL) + status = catch(:halt_check) do ce_branch_compat_check! delete_ee_branches_locally! ee_branch_presence_check! + + step("Checking out #{ee_branch_found}", %W[git checkout -b #{ee_branch_found} canonical-ee/#{ee_branch_found}]) + generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, remote: 'canonical-ee') ee_branch_compat_check! end @@ -56,6 +62,13 @@ module Gitlab private + def add_remote(name, url) + step( + "Adding the #{name} remote (#{url})", + %W[git remote add #{name} #{url}] + ) + end + def ensure_ee_repo if Dir.exist?(ee_repo_dir) step("#{ee_repo_dir} already exists") @@ -71,14 +84,14 @@ module Gitlab FileUtils.mkdir_p(patches_dir) end - def generate_patch(branch, patch_path) + def generate_patch(branch:, patch_path:, remote:) FileUtils.rm(patch_path, force: true) - find_merge_base_with_master(branch: branch) + find_merge_base_with_master(branch: branch, master_remote: remote) step( - "Generating the patch against origin/master in #{patch_path}", - %w[git diff --binary origin/master...HEAD] + "Generating the patch against #{remote}/master in #{patch_path}", + %W[git diff --binary #{remote}/master...#{branch}] ) do |output, status| throw(:halt_check, :ko) unless status.zero? @@ -89,21 +102,21 @@ module Gitlab end def ce_branch_compat_check! - if check_patch(ce_patch_full_path).zero? + if check_patch(ce_patch_full_path, remote: 'canonical-ce').zero? puts applies_cleanly_msg(ce_branch) throw(:halt_check) end end def ee_branch_presence_check! - _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch origin #{ee_branch_prefix}]) + _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch canonical-ee #{ee_branch_prefix}]) if status.zero? @ee_branch_found = ee_branch_prefix return end - _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}]) + _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch canonical-ee #{ee_branch_suffix}]) if status.zero? @ee_branch_found = ee_branch_suffix @@ -116,11 +129,7 @@ module Gitlab end def ee_branch_compat_check! - step("Checking out origin/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} FETCH_HEAD]) - - generate_patch(ee_branch_found, ee_patch_full_path) - - unless check_patch(ee_patch_full_path).zero? + unless check_patch(ee_patch_full_path, remote: 'canonical-ee').zero? puts puts ee_branch_doesnt_apply_cleanly_msg @@ -131,10 +140,9 @@ module Gitlab puts applies_cleanly_msg(ee_branch_found) end - def check_patch(patch_path) + def check_patch(patch_path, remote:) step("Checking out master", %w[git checkout master]) - step("Resetting to latest master", %w[git reset --hard origin/master]) - step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo_url} #{ce_branch}]) + step("Resetting to latest master", %W[git reset --hard #{remote}/master]) step( "Checking if #{patch_path} applies cleanly to EE/master", # Don't use --check here because it can result in a 0-exit status even @@ -171,10 +179,10 @@ module Gitlab command(%W[git branch --delete --force #{ee_branch_suffix}]) end - def merge_base_found? + def merge_base_found?(master_remote:, branch:) step( - "Finding merge base with master", - %w[git merge-base origin/master HEAD] + "Finding merge base with #{master_remote}/master", + %W[git merge-base #{master_remote}/master #{branch}] ) do |output, status| if status.zero? puts "Merge base was found: #{output}" @@ -183,7 +191,7 @@ module Gitlab end end - def find_merge_base_with_master(branch:) + def find_merge_base_with_master(branch:, master_remote:) # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 @@ -192,19 +200,19 @@ module Gitlab depth += Math.exp(factor).to_i # Repository is initially cloned with a depth of 20 so we need to fetch # deeper in the case the branch has more than 20 commits on top of master - fetch(branch: branch, depth: depth) - fetch(branch: 'master', depth: depth, remote: DEFAULT_CE_PROJECT_URL) + fetch(branch: branch, depth: depth, remote: 'origin') + fetch(branch: 'master', depth: depth, remote: master_remote) - merge_base_found? + merge_base_found?(master_remote: master_remote, branch: branch) end - raise "\n#{branch} is too far behind master, please rebase it!\n" unless success + raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success end def fetch(branch:, depth:, remote: 'origin') step( "Fetching deeper...", - %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/origin/#{branch}] + %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/#{remote}/#{branch}] ) do |output, status| raise "Fetch failed: #{output}" unless status.zero? end @@ -304,8 +312,8 @@ module Gitlab 1. Create a new branch from master and cherry-pick your CE commits # In the EE repo - $ git fetch origin - $ git checkout -b #{ee_branch_prefix} origin/master + $ git fetch #{EE_REPO_URL} master + $ git checkout -b #{ee_branch_prefix} FETCH_HEAD $ git fetch #{ce_repo_url} #{ce_branch} $ git cherry-pick SHA # Repeat for all the commits you want to pick @@ -314,10 +322,9 @@ module Gitlab 2. Apply your branch's patch to EE # In the EE repo - $ git fetch origin master - $ git checkout -b #{ee_branch_prefix} origin/master - $ wget #{patch_url} - $ git apply --3way #{ce_patch_name} + $ git fetch #{EE_REPO_URL} master + $ git checkout -b #{ee_branch_prefix} FETCH_HEAD + $ wget #{patch_url} && git apply --3way #{ce_patch_name} At this point you might have conflicts such as: -- cgit v1.2.1 From bf77f1cd10e424be25ef9b155fc1b2a48d7556c2 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Wed, 22 Nov 2017 19:13:47 +0100 Subject: Force disable Prometheus metrics Until https://gitlab.com/gitlab-org/prometheus-client-mmap/merge_requests/11 is ready, Prometheus metrics will not work and cause issues such as #40457. --- lib/gitlab/metrics/prometheus.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 09103b4ca2d..4f165d12a94 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -17,9 +17,9 @@ module Gitlab end def prometheus_metrics_enabled? - return @prometheus_metrics_enabled if defined?(@prometheus_metrics_enabled) - - @prometheus_metrics_enabled = prometheus_metrics_enabled_unmemoized + # force disable prometheus_metrics until + # https://gitlab.com/gitlab-org/prometheus-client-mmap/merge_requests/11 is ready + false end def registry -- cgit v1.2.1 From 11d0787961b33efdbfdad9bf0bbc48afecc3cc53 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Wed, 22 Nov 2017 17:46:40 +0000 Subject: Speed up Unicorn specs by using a dummy Rack application instead of GitLab --- lib/gitlab/path_regex.rb | 1 - lib/tasks/brakeman.rake | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 9a91f8bf96a..7e5dfd33502 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -51,7 +51,6 @@ module Gitlab slash-command-logo.png snippets u - unicorn_test unsubscribes uploads users diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 99b3168d9eb..2301ec9b228 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' -- cgit v1.2.1 From eceec26795b5227110f2209acbcc6aa6a21ba108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 22 Nov 2017 21:43:27 +0000 Subject: Try to always find the merge base using `origin/#{branch}` instead of just `#{branch}` --- lib/gitlab/ee_compat_check.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 3a6165f504c..ac730278ca7 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -182,7 +182,7 @@ module Gitlab def merge_base_found?(master_remote:, branch:) step( "Finding merge base with #{master_remote}/master", - %W[git merge-base #{master_remote}/master #{branch}] + %W[git merge-base #{master_remote}/master origin/#{branch}] ) do |output, status| if status.zero? puts "Merge base was found: #{output}" -- cgit v1.2.1 From f5ab0f52a10d23f300338a29389f78cf8cda1136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 22 Nov 2017 22:07:57 +0000 Subject: Generate the patch against `origin/#{branch}` instead of just `#{branch}` --- lib/gitlab/ee_compat_check.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index ac730278ca7..66b2c60e8eb 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -91,7 +91,7 @@ module Gitlab step( "Generating the patch against #{remote}/master in #{patch_path}", - %W[git diff --binary #{remote}/master...#{branch}] + %W[git diff --binary #{remote}/master...origin/#{branch}] ) do |output, status| throw(:halt_check, :ko) unless status.zero? -- cgit v1.2.1 From 3262378e240880b8dcadd83b0dd7fca2c618f146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 22 Nov 2017 23:30:31 +0100 Subject: In EeCompatCheck, always reset to canonical-ee/master when applying a patch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/ee_compat_check.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 66b2c60e8eb..4a9d3e52fae 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -102,7 +102,7 @@ module Gitlab end def ce_branch_compat_check! - if check_patch(ce_patch_full_path, remote: 'canonical-ce').zero? + if check_patch(ce_patch_full_path).zero? puts applies_cleanly_msg(ce_branch) throw(:halt_check) end @@ -129,7 +129,7 @@ module Gitlab end def ee_branch_compat_check! - unless check_patch(ee_patch_full_path, remote: 'canonical-ee').zero? + unless check_patch(ee_patch_full_path).zero? puts puts ee_branch_doesnt_apply_cleanly_msg @@ -140,9 +140,9 @@ module Gitlab puts applies_cleanly_msg(ee_branch_found) end - def check_patch(patch_path, remote:) + def check_patch(patch_path) step("Checking out master", %w[git checkout master]) - step("Resetting to latest master", %W[git reset --hard #{remote}/master]) + step("Resetting to latest master", %w[git reset --hard canonical-ee/master]) step( "Checking if #{patch_path} applies cleanly to EE/master", # Don't use --check here because it can result in a 0-exit status even -- cgit v1.2.1 From 7df1cb528e5d977f5f18b1c82433b3c76220d6db Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Sat, 18 Nov 2017 02:13:45 +0800 Subject: Move identical merged branch check to merged_branch_names --- lib/api/entities.rb | 6 +++++- lib/gitlab/git/repository.rb | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 16ae99b5c6c..acfd4a1ef4d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -242,7 +242,11 @@ module API end expose :merged do |repo_branch, options| - options[:project].repository.merged_to_root_ref?(repo_branch, options[:merged_branch_names]) + if options[:merged_branch_names] + options[:merged_branch_names].include?(repo_branch.name) + else + options[:project].repository.merged_to_root_ref?(repo_branch) + end end expose :protected do |repo_branch, options| diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index dcca20c75ef..ff408c0c4dd 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1243,11 +1243,21 @@ module Gitlab sort_branches(branches, sort_by) end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 def git_merged_branch_names(branch_names = []) - lines = run_git(['branch', '--merged', root_ref] + branch_names) - .first.lines + root_sha = find_branch(root_ref).target - lines.map(&:strip) + git_arguments = + %W[branch --merged #{root_sha} + --format=%(refname:short)\ %(objectname)] + branch_names + + lines = run_git(git_arguments).first.lines + + lines.each_with_object([]) do |line, branches| + name, sha = line.strip.split(' ', 2) + + branches << name if sha != root_sha + end end def log_using_shell?(options) -- cgit v1.2.1 From 4cfcc97544c231c2baf8dc3ab232ed394355b62c Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Thu, 23 Nov 2017 10:48:57 +0000 Subject: Fix encoding bugs in Gitlab::Git::User --- lib/gitlab/encoding_helper.rb | 4 ++++ lib/gitlab/git/user.rb | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 99dfee3dd9b..582028493e9 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -17,6 +17,10 @@ module Gitlab return nil unless message.respond_to?(:force_encoding) return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + if message.respond_to?(:frozen?) && message.frozen? + message = message.dup + end + message.force_encoding("UTF-8") return message if message.valid_encoding? diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index e6b61417de1..e573cd0e143 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -8,7 +8,12 @@ module Gitlab end def self.from_gitaly(gitaly_user) - new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new( + gitaly_user.gl_username, + Gitlab::EncodingHelper.encode!(gitaly_user.name), + Gitlab::EncodingHelper.encode!(gitaly_user.email), + gitaly_user.gl_id + ) end def initialize(username, name, email, gl_id) @@ -23,7 +28,7 @@ module Gitlab end def to_gitaly - Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email) + Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b) end end end -- cgit v1.2.1 From e826c5d0917a7fe2225fb6ba0862bc56c1ef3fc2 Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Wed, 22 Nov 2017 14:20:35 +0100 Subject: Fix link text from group context --- lib/banzai/filter/abstract_reference_filter.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 9fef386de16..8975395aff1 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -213,7 +213,8 @@ module Banzai end def object_link_text(object, matches) - text = object.reference_link_text(context[:project]) + parent = context[:project] || context[:group] + text = object.reference_link_text(parent) extras = object_link_text_extras(object, matches) text += " (#{extras.join(", ")})" if extras.any? -- cgit v1.2.1 From 991bf24ec8890eca248a00deb4f33f309c9ffb83 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Wed, 15 Nov 2017 17:22:18 +0000 Subject: Use latest_merge_request_diff association Compared to the merge_request_diff association: 1. It's simpler to query. The query uses a foreign key to the merge_request_diffs table, so no ordering is necessary. 2. It's faster for preloading. The merge_request_diff association has to load every diff for the MRs in the set, then discard all but the most recent for each. This association means that Rails can just query for N diffs from N MRs. 3. It's more complicated to update. This is a bidirectional foreign key, so we need to update two tables when adding a diff record. This also means we need to handle this as a special case when importing a GitLab project. There is some juggling with this association in the merge request model: * `MergeRequest#latest_merge_request_diff` is _always_ the latest diff. * `MergeRequest#merge_request_diff` reuses `MergeRequest#latest_merge_request_diff` unless: * Arguments are passed. These are typically to force-reload the association. * It doesn't exist. That means we might be trying to implicitly create a diff. This only seems to happen in specs. * The association is already loaded. This is important for the reasons explained in the comment, which I'll reiterate here: if we a) load a non-latest diff, then b) get its `merge_request`, then c) get that MR's `merge_request_diff`, we should get the diff we loaded in c), even though that's not the latest diff. Basically, `MergeRequest#merge_request_diff` is the latest diff in most cases, but not quite all. --- lib/api/merge_requests.rb | 2 +- lib/gitlab/import_export/project_tree_restorer.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 726f09e3669..5b4642a2f57 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -21,7 +21,7 @@ module API return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels, :timelogs) + .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) end params :merge_requests_params do diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 639f4f0c3f0..c518943be59 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -60,6 +60,8 @@ module Gitlab end end + @project.merge_requests.set_latest_merge_request_diff_ids! + @saved end -- cgit v1.2.1 From fceffe4dca9af5ab3ffaa2e110c5fb4bad254950 Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Wed, 22 Nov 2017 15:16:08 +0000 Subject: Renamed ProtectedBranches::ApiUpdateService to LegacyApiUpdateService --- lib/api/branches.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/branches.rb b/lib/api/branches.rb index cdef1b546a9..0791a110c39 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -81,9 +81,9 @@ module API service_args = [user_project, current_user, protected_branch_params] protected_branch = if protected_branch - ::ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + ::ProtectedBranches::LegacyApiUpdateService.new(*service_args).execute(protected_branch) else - ::ProtectedBranches::ApiCreateService.new(*service_args).execute + ::ProtectedBranches::LegacyApiCreateService.new(*service_args).execute end if protected_branch.valid? -- cgit v1.2.1 From b3331cf3dfe3c5081bf448279456701009d17231 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 23 Nov 2017 13:56:51 +0100 Subject: Added Rubocop config for background migrations This adds a Rubocop configuration file specific to lib/gitlab/background_migrations. This configuration will be used to (hopefully) make reviewing background migrations easier by enforcing stricter rules compared to the rest of GitLab. Because this configuration is directory specific it will only affect background migrations. --- lib/gitlab/background_migration/.rubocop.yml | 52 ++++++++++++++++++++++ .../create_fork_network_memberships_range.rb | 4 ++ .../create_gpg_key_subkeys_from_gpg_keys.rb | 4 ++ .../delete_conflicting_redirect_routes_range.rb | 4 ++ .../deserialize_merge_request_diffs_and_commits.rb | 6 +++ .../migrate_build_stage_id_reference.rb | 3 ++ .../migrate_events_to_push_event_payloads.rb | 4 ++ .../background_migration/migrate_stage_status.rb | 4 ++ .../migrate_system_uploads_to_new_folder.rb | 4 ++ .../move_personal_snippet_files.rb | 4 ++ .../normalize_ldap_extern_uids_range.rb | 7 +++ .../populate_fork_networks_range.rb | 5 +++ ..._merge_requests_latest_merge_request_diff_id.rb | 3 ++ 13 files changed, 104 insertions(+) create mode 100644 lib/gitlab/background_migration/.rubocop.yml (limited to 'lib') diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml new file mode 100644 index 00000000000..8242821cedc --- /dev/null +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -0,0 +1,52 @@ +# For background migrations we define a custom set of rules to make it less +# difficult to review these migrations. To reduce the complexity of these +# migrations some rules may be stricter than the defaults set in the root +# .rubocop.yml file. +--- +inherit_from: ../../../.rubocop.yml + +Metrics/AbcSize: + Enabled: true + Max: 30 + Details: > + Code that involves a lot of branches can be very hard to wrap your head + around. + +Metrics/PerceivedComplexity: + Enabled: true + +Metrics/LineLength: + Enabled: true + Details: > + Long lines are very hard to read and make it more difficult to review + changes. + +Metrics/MethodLength: + Enabled: true + Max: 30 + Details: > + Long methods can be very hard to review. Consider splitting this method up + into separate methods. + +Metrics/ClassLength: + Enabled: true + Details: > + Long classes can be very hard to review. Consider splitting this class up + into multiple classes. + +Metrics/BlockLength: + Enabled: true + Details: > + Long blocks can be hard to read. Consider splitting the code into separate + methods. + +Style/Documentation: + Enabled: true + Details: > + Adding documentation makes it easier to figure out what a migration is + supposed to do. + +Style/FrozenStringLiteralComment: + Enabled: true + Details: >- + This removes the need for calling "freeze", reducing noise in the code. diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 67a39d28944..03b17b319fa 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class CreateForkNetworkMembershipsRange diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb index e94719db72e..c2bf42f846d 100644 --- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys class GpgKey < ActiveRecord::Base self.table_name = 'gpg_keys' diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb index b1411be3016..a1af045a71f 100644 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class DeleteConflictingRedirectRoutesRange diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 380802258f5..fd5cbf76e47 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -1,3 +1,9 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class DeserializeMergeRequestDiffsAndCommits diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index 91540127ea9..0a8a4313cd5 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateBuildStageIdReference diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 432f7c3e706..84ac00f1a5c 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration # Class that migrates events for the new push event payloads setup. All diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index b1ff0900709..0e5c7f092f2 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateStageStatus diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb index 0881244ed49..7f243073fd0 100644 --- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateSystemUploadsToNewFolder diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb index 07cec96bcc3..a4ef51fd0e8 100644 --- a/lib/gitlab/background_migration/move_personal_snippet_files.rb +++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MovePersonalSnippetFiles diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb index bc53e6d7f94..85749366bfd 100644 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -1,3 +1,10 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/ClassLength +# rubocop:disable Metrics/BlockLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class NormalizeLdapExternUidsRange diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb index 2ef3a207dd3..f8508b5fbdf 100644 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -1,3 +1,8 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class PopulateForkNetworksRange diff --git a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb index 7e109e96e73..dcac355e1b0 100644 --- a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb +++ b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class PopulateMergeRequestsLatestMergeRequestDiffId -- cgit v1.2.1 From 257fd5713485a05460a9170190100643199a7e48 Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Thu, 23 Nov 2017 13:16:14 +0000 Subject: Allow password authentication to be disabled entirely --- lib/api/entities.rb | 5 ++++- lib/api/settings.rb | 13 +++++++++---- lib/api/v3/entities.rb | 4 ++-- lib/api/v3/settings.rb | 8 +++++--- lib/gitlab/auth.rb | 14 ++++++++++---- lib/gitlab/usage_data.rb | 2 +- 6 files changed, 31 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 16ae99b5c6c..e45c87e4f4f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -763,7 +763,10 @@ module API expose(:default_project_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_project_visibility) } expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) } expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) } - expose :password_authentication_enabled, as: :signin_enabled + + # support legacy names, can be removed in v5 + expose :password_authentication_enabled_for_web, as: :password_authentication_enabled + expose :password_authentication_enabled_for_web, as: :signin_enabled end class Release < Grape::Entity diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 851b226e9e5..06373fe5069 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -44,9 +44,11 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - mutually_exclusive :password_authentication_enabled, :signin_enabled + optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled + optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)' optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -135,8 +137,11 @@ module API put "application/settings" do attrs = declared_params(include_missing: false) + # support legacy names, can be removed in v5 if attrs.has_key?(:signin_enabled) - attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:signin_enabled) + elsif attrs.has_key?(:password_authentication_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end if current_settings.update_attributes(attrs) diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index afdd7b83998..c17b6f45ed8 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -172,8 +172,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :password_authentication_enabled - expose :password_authentication_enabled, as: :signin_enabled + expose :password_authentication_enabled_for_web, as: :password_authentication_enabled + expose :password_authentication_enabled_for_web, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb index 202011cfcbe..9b4ab7630fb 100644 --- a/lib/api/v3/settings.rb +++ b/lib/api/v3/settings.rb @@ -44,8 +44,8 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do @@ -131,7 +131,9 @@ module API attrs = declared_params(include_missing: false) if attrs.has_key?(:signin_enabled) - attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:signin_enabled) + elsif attrs.has_key?(:password_authentication_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end if current_settings.update_attributes(attrs) diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index cbbc51db99e..9670207a105 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -34,7 +34,7 @@ module Gitlab rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + return result if result.success? || authenticate_using_internal_or_ldap_password? # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts @@ -45,6 +45,10 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless authenticate_using_internal_or_ldap_password? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -52,10 +56,8 @@ module Gitlab # LDAP users are only authenticated via LDAP if user.nil? || user.ldap_user? # Second chance - try LDAP authentication - return unless Gitlab::LDAP::Config.enabled? - Gitlab::LDAP::Authentication.login(login, password) - else + elsif current_application_settings.password_authentication_enabled_for_git? user if user.active? && user.valid_password?(password) end end @@ -84,6 +86,10 @@ module Gitlab private + def authenticate_using_internal_or_ldap_password? + current_application_settings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + end + def service_request_check(login, password, project) matched_login = /(?^[a-zA-Z]*-ci)-token$/.match(login) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 112d4939582..2adcc9809b3 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -79,7 +79,7 @@ module Gitlab def features_usage_data_ce { - signup: current_application_settings.signup_enabled?, + signup: current_application_settings.allow_signup?, ldap: Gitlab.config.ldap.enabled, gravatar: current_application_settings.gravatar_enabled?, omniauth: Gitlab.config.omniauth.enabled, -- cgit v1.2.1 From d0a08ab888db33437c7df4eb37b5805757a6dce4 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Sat, 18 Nov 2017 07:26:07 +0100 Subject: Improve storage migration rake task --- lib/tasks/gitlab/storage.rake | 85 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index e05be4a3405..8ac73bc8ff2 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -2,10 +2,10 @@ namespace :gitlab do namespace :storage do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do - legacy_projects_count = Project.with_legacy_storage.count + legacy_projects_count = Project.with_unmigrated_storage.count if legacy_projects_count == 0 - puts 'There are no projects using legacy storage. Nothing to do!' + puts 'There are no projects requiring storage migration. Nothing to do!' next end @@ -23,22 +23,42 @@ namespace :gitlab do desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' task legacy_projects: :environment do - projects_summary(Project.with_legacy_storage) + relation_summary('projects', Project.without_storage_feature(:repository)) end desc 'Gitlab | Storage | List existing projects using Legacy Storage' task list_legacy_projects: :environment do - projects_list(Project.with_legacy_storage) + projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository)) end desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' task hashed_projects: :environment do - projects_summary(Project.with_hashed_storage) + relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository)) end desc 'Gitlab | Storage | List existing projects using Hashed Storage' task list_hashed_projects: :environment do - projects_list(Project.with_hashed_storage) + projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository)) + end + + desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage' + task legacy_attachments: :environment do + relation_summary('attachments using Legacy Storage', legacy_attachments_relation) + end + + desc 'Gitlab | Storage | List existing project attachments using Legacy Storage' + task list_legacy_attachments: :environment do + attachments_list('attachments using Legacy Storage', legacy_attachments_relation) + end + + desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage' + task hashed_attachments: :environment do + relation_summary('attachments using Hashed Storage', hashed_attachments_relation) + end + + desc 'Gitlab | Storage | List existing project attachments using Hashed Storage' + task list_hashed_attachments: :environment do + attachments_list('attachments using Hashed Storage', hashed_attachments_relation) end def batch_size @@ -46,29 +66,43 @@ namespace :gitlab do end def project_id_batches(&block) - Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) yield ids.min, ids.max end end - def projects_summary(relation) - projects_count = relation.count - puts "* Found #{projects_count} projects".color(:green) + def legacy_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def hashed_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def relation_summary(relation_name, relation) + relation_count = relation.count + puts "* Found #{relation_count} #{relation_name}".color(:green) - projects_count + relation_count end - def projects_list(relation) - projects_count = projects_summary(relation) + def projects_list(relation_name, relation) + relation_count = relation_summary(relation_name, relation) projects = relation.with_route limit = ENV.fetch('LIMIT', 500).to_i - return unless projects_count > 0 + return unless relation_count > 0 - puts " ! Displaying first #{limit} projects..." if projects_count > limit + puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit counter = 0 projects.find_in_batches(batch_size: batch_size) do |batch| @@ -81,5 +115,26 @@ namespace :gitlab do end end end + + def attachments_list(relation_name, relation) + relation_count = relation_summary(relation_name, relation) + + limit = ENV.fetch('LIMIT', 500).to_i + + return unless relation_count > 0 + + puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit + + counter = 0 + relation.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |upload| + counter += 1 + + puts " - #{upload.path} (id: #{upload.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end end end -- cgit v1.2.1 From 392cc887097bfd0c28b547700b9a67c32cf14e69 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Fri, 17 Nov 2017 02:43:55 +0100 Subject: Add new API endpoint - get a namespace by ID --- lib/api/namespaces.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'lib') diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index f1eaff6b0eb..21dc5009d0e 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -19,6 +19,30 @@ module API present paginate(namespaces), with: Entities::Namespace, current_user: current_user end + + desc 'Get a namespace by ID' do + success Entities::Namespace + end + params do + requires :id, type: Integer, desc: "Namespace's ID" + end + get ':id' do + namespace = Namespace.find(params[:id]) + authenticate_get_namespace!(namespace) + + present namespace, with: Entities::Namespace, current_user: current_user + end + end + + helpers do + def authenticate_get_namespace!(namespace) + return if current_user.admin? + forbidden!('No access granted') unless user_can_access_namespace?(namespace) + end + + def user_can_access_namespace?(namespace) + namespace.has_owner?(current_user) + end end end end -- cgit v1.2.1 From 453b17809395fda045f5685268cae58c1dceb881 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 23 Nov 2017 12:41:29 +0100 Subject: Fix pulling and pushing using a personal access token with the sudo scope --- lib/gitlab/auth.rb | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index cbbc51db99e..0e7958ef90f 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -128,7 +128,7 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) if token && valid_scoped_token?(token, available_scopes) - Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scope(token.scopes)) + Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end @@ -140,10 +140,15 @@ module Gitlab AccessTokenValidationService.new(token).include_any_scope?(scopes) end - def abilities_for_scope(scopes) - scopes.map do |scope| - self.public_send(:"#{scope}_scope_authentication_abilities") # rubocop:disable GitlabSecurity/PublicSend - end.flatten.uniq + def abilities_for_scopes(scopes) + abilities_by_scope = { + api: full_authentication_abilities, + read_registry: [:read_container_image] + } + + scopes.flat_map do |scope| + abilities_by_scope.fetch(scope.to_sym, []) + end.uniq end def lfs_token_check(login, password, project) @@ -222,16 +227,6 @@ module Gitlab :admin_container_image ] end - alias_method :api_scope_authentication_abilities, :full_authentication_abilities - - def read_registry_scope_authentication_abilities - [:read_container_image] - end - - # The currently used auth method doesn't allow any actions for this scope - def read_user_scope_authentication_abilities - [] - end def available_scopes(current_user = nil) scopes = API_SCOPES + registry_scopes -- cgit v1.2.1 From 0e6beaf50c9233ca03083691856dea2883f71773 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 15 Nov 2017 16:46:08 +0100 Subject: Clean up repository fetch and mirror methods --- lib/gitlab/git/repository.rb | 10 +++-- lib/gitlab/git/repository_mirroring.rb | 46 +++++++++++++--------- lib/gitlab/github_import.rb | 4 ++ .../github_import/importer/repository_importer.rb | 17 +------- lib/gitlab/legacy_github_import/importer.rb | 4 ++ 5 files changed, 43 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index dcca20c75ef..69897bf1402 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1150,10 +1150,12 @@ module Gitlab @has_visible_content = has_local_branches? end - def fetch(remote = 'origin') - args = %W(#{Gitlab.config.git.bin_path} fetch #{remote}) - - popen(args, @path).last.zero? + # Like all public `Gitlab::Git::Repository` methods, this method is part + # of `Repository`'s interface through `method_missing`. + # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and + # takes some extra attributes, so we qualify this method name to prevent confusion. + def fetch_remote_without_shell(remote = 'origin') + run_git(['fetch', remote]).last.zero? end def blob_at(sha, path) diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index 4500482d68f..49e2b833fa7 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -1,24 +1,26 @@ module Gitlab module Git module RepositoryMirroring - IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze - IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze - MIRROR_REMOTE = 'mirror'.freeze + FETCH_REFS = { + # `:all` is used to define repository as equivalent as "git clone --mirror" + all: '+refs/*:refs/*', + heads: '+refs/heads/*:refs/heads/*', + tags: '+refs/tags/*:refs/tags/*' + }.freeze RemoteError = Class.new(StandardError) - def set_remote_as_mirror(remote_name) - # This is used to define repository as equivalent as "git clone --mirror" - rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*' - rugged.config["remote.#{remote_name}.mirror"] = true - rugged.config["remote.#{remote_name}.prune"] = true - end + def set_remote_as_mirror(remote_name, fetch_refs: :all) + Array(fetch_refs).each_with_index do |fetch_ref, i| + fetch_ref = FETCH_REFS[fetch_ref] || fetch_ref - def set_import_remote_as_mirror(remote_name) - # Add first fetch with Rugged so it does not create its own. - rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS - - add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) + # Add first fetch with Rugged so it does not create its own. + if i == 0 + rugged.config["remote.#{remote_name}.fetch"] = fetch_ref + else + add_remote_fetch_config(remote_name, fetch_ref) + end + end rugged.config["remote.#{remote_name}.mirror"] = true rugged.config["remote.#{remote_name}.prune"] = true @@ -28,11 +30,17 @@ module Gitlab run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) end - def fetch_mirror(url) - add_remote(MIRROR_REMOTE, url) - set_remote_as_mirror(MIRROR_REMOTE) - fetch(MIRROR_REMOTE) - remove_remote(MIRROR_REMOTE) + # Like all public `Gitlab::Git::Repository` methods, this method is part + # of `Repository`'s interface through `method_missing`. + # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and + # takes some extra attributes, so we qualify this method name to prevent confusion. + def fetch_as_mirror_without_shell(url) + remote_name = "tmp-#{SecureRandom.hex}" + add_remote(remote_name, url) + set_remote_as_mirror(remote_name) + fetch_remote_without_shell(remote_name) + ensure + remove_remote(remote_name) if remote_name end def remote_tags(remote) diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index d2ae4c1255e..427dab82e19 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -1,5 +1,9 @@ module Gitlab module GithubImport + def self.fetch_refs + [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head'] + end + def self.new_client_for(project, token: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 0b67fc8db73..7b1994de012 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -45,27 +45,14 @@ module Gitlab def import_repository project.ensure_repository - configure_repository_remote - - project.repository.fetch_remote('github', forced: true) + fetch_refs = Gitlab::GithubImport.fetch_refs + project.repository.fetch_as_mirror(project.import_url, fetch_refs: fetch_refs, forced: true, remote_name: 'github') true rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e fail_import("Failed to import the repository: #{e.message}") end - def configure_repository_remote - return if project.repository.remote_exists?('github') - - project.repository.add_remote('github', project.import_url) - project.repository.set_import_remote_as_mirror('github') - - project.repository.add_remote_fetch_config( - 'github', - '+refs/pull/*/head:refs/merge-requests/*/head' - ) - end - def import_wiki_repository wiki_path = "#{project.disk_path}.wiki" wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 4d096e5a741..0e5425d01fd 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -3,6 +3,10 @@ module Gitlab class Importer include Gitlab::ShellAdapter + def self.fetch_refs + Gitlab::GithubImport.fetch_refs + end + attr_reader :errors, :project, :repo, :repo_url def initialize(project) -- cgit v1.2.1 From 7a1e93d35b7280db8bc4128862c86223d76a8d6d Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 23 Nov 2017 16:51:55 +0100 Subject: Rename fetch_refs to refmap --- lib/gitlab/git/repository_mirroring.rb | 35 +++++++++++----------- lib/gitlab/github_import.rb | 2 +- .../github_import/importer/repository_importer.rb | 4 +-- lib/gitlab/legacy_github_import/importer.rb | 4 +-- 4 files changed, 23 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index 49e2b833fa7..392bef69e80 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -1,36 +1,37 @@ module Gitlab module Git module RepositoryMirroring - FETCH_REFS = { - # `:all` is used to define repository as equivalent as "git clone --mirror" - all: '+refs/*:refs/*', + REFMAPS = { + # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror` + all_refs: '+refs/*:refs/*', heads: '+refs/heads/*:refs/heads/*', tags: '+refs/tags/*:refs/tags/*' }.freeze RemoteError = Class.new(StandardError) - def set_remote_as_mirror(remote_name, fetch_refs: :all) - Array(fetch_refs).each_with_index do |fetch_ref, i| - fetch_ref = FETCH_REFS[fetch_ref] || fetch_ref - - # Add first fetch with Rugged so it does not create its own. - if i == 0 - rugged.config["remote.#{remote_name}.fetch"] = fetch_ref - else - add_remote_fetch_config(remote_name, fetch_ref) - end - end + def set_remote_as_mirror(remote_name, refmap: :all_refs) + set_remote_refmap(remote_name, refmap) rugged.config["remote.#{remote_name}.mirror"] = true rugged.config["remote.#{remote_name}.prune"] = true end - def add_remote_fetch_config(remote_name, refspec) - run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + def set_remote_refmap(remote_name, refmap) + Array(refmap).each_with_index do |refspec, i| + refspec = REFMAPS[refspec] || refspec + + # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it. + # To make sure we start from scratch, we set the first using rugged, and use `git` for any others + if i == 0 + rugged.config["remote.#{remote_name}.fetch"] = refspec + else + run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + end + end end - # Like all public `Gitlab::Git::Repository` methods, this method is part + # Like all_refs public `Gitlab::Git::Repository` methods, this method is part # of `Repository`'s interface through `method_missing`. # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and # takes some extra attributes, so we qualify this method name to prevent confusion. diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 427dab82e19..65b5e30c70f 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - def self.fetch_refs + def self.refmap [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head'] end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 7b1994de012..9cf2e7fd871 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -45,8 +45,8 @@ module Gitlab def import_repository project.ensure_repository - fetch_refs = Gitlab::GithubImport.fetch_refs - project.repository.fetch_as_mirror(project.import_url, fetch_refs: fetch_refs, forced: true, remote_name: 'github') + refmap = Gitlab::GithubImport.refmap + project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true, remote_name: 'github') true rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 0e5425d01fd..0526ef9eb13 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -3,8 +3,8 @@ module Gitlab class Importer include Gitlab::ShellAdapter - def self.fetch_refs - Gitlab::GithubImport.fetch_refs + def self.refmap + Gitlab::GithubImport.refmap end attr_reader :errors, :project, :repo, :repo_url -- cgit v1.2.1 From dfbfd3c7d7d4677ac99a7f8147a673911e8d4e98 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 23 Nov 2017 14:32:16 +0100 Subject: Allow request namespace by ID or path --- lib/api/helpers.rb | 22 ++++++++++++++++++++++ lib/api/namespaces.rb | 18 ++---------------- 2 files changed, 24 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index b26c61ab8da..52ac416f9ad 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -50,6 +50,10 @@ module API initial_current_user != current_user end + def user_namespace + @user_namespace ||= find_namespace!(params[:id]) + end + def user_group @group ||= find_group!(params[:id]) end @@ -112,6 +116,24 @@ module API end end + def find_namespace(id) + if id.to_s =~ /^\d+$/ + Namespace.find_by(id: id) + else + Namespace.find_by_full_path(id) + end + end + + def find_namespace!(id) + namespace = find_namespace(id) + + if can?(current_user, :admin_namespace, namespace) + namespace + else + not_found!('Namespace') + end + end + def find_project_label(id) label = available_labels.find_by_id(id) || available_labels.find_by_title(id) label || not_found!('Label') diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 21dc5009d0e..32b77aedba8 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -24,24 +24,10 @@ module API success Entities::Namespace end params do - requires :id, type: Integer, desc: "Namespace's ID" + requires :id, type: String, desc: "Namespace's ID or path" end get ':id' do - namespace = Namespace.find(params[:id]) - authenticate_get_namespace!(namespace) - - present namespace, with: Entities::Namespace, current_user: current_user - end - end - - helpers do - def authenticate_get_namespace!(namespace) - return if current_user.admin? - forbidden!('No access granted') unless user_can_access_namespace?(namespace) - end - - def user_can_access_namespace?(namespace) - namespace.has_owner?(current_user) + present user_namespace, with: Entities::Namespace, current_user: current_user end end end -- cgit v1.2.1 From 97f966c445c0c2191a8017aa981a34737b9adf56 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 23 Nov 2017 15:47:15 +0100 Subject: Introduce :read_namespace access policy for namespace and group --- lib/api/helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 52ac416f9ad..686bf7a3c2b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -127,7 +127,7 @@ module API def find_namespace!(id) namespace = find_namespace(id) - if can?(current_user, :admin_namespace, namespace) + if can?(current_user, :read_namespace, namespace) namespace else not_found!('Namespace') -- cgit v1.2.1 From fdd7a4cb1bb8f25dd2c8c6b4762d422e138d809a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 23 Nov 2017 17:04:35 +0000 Subject: Fix hashed storage for attachments bugs --- lib/gitlab/import_export/uploads_saver.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index f9ae5079d7c..627a487d577 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -24,8 +24,7 @@ module Gitlab end def uploads_path - # TODO: decide what to do with uploads. We will use UUIDs here too? - File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) + FileUploader.dynamic_path_segment(@project) end end end -- cgit v1.2.1 From cdcbeaccbec09fefe81b7b90badacf5308751ce9 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 23 Nov 2017 00:26:50 +0100 Subject: Move prometheus middle ware to prometheus initialized. --- lib/api/settings.rb | 3 +++ lib/gitlab/metrics/method_call.rb | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 851b226e9e5..a43f8d0497c 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -66,6 +66,9 @@ module API optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' + given prometheus_metrics_enabled: ->(val) { val } do + requires :prometheus_metrics_method_instrumentation_enabled, type: Boolean, desc: 'Enable method call instrumentation' + end optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' given metrics_enabled: ->(val) { val } do requires :metrics_host, type: String, desc: 'The InfluxDB host' diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 90235095306..5177d953795 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -59,8 +59,10 @@ module Gitlab @cpu_time += cpu_time @call_count += 1 - self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) - self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0) + if prometheus_enabled? + self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) + self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0) + end retval end @@ -83,6 +85,10 @@ module Gitlab def above_threshold? real_time >= Metrics.method_call_threshold end + + def prometheus_enabled? + @prometheus_enabled ||= current_application_settings(:prometheus_metrics_method_instrumentation_enabled) + end end end end -- cgit v1.2.1 From 9884c0f4a0ea634fe08235ef0a0c7f997f19a782 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 23 Nov 2017 00:37:55 +0100 Subject: Reenable prometheus metrics --- lib/gitlab/metrics/prometheus.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 4f165d12a94..09103b4ca2d 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -17,9 +17,9 @@ module Gitlab end def prometheus_metrics_enabled? - # force disable prometheus_metrics until - # https://gitlab.com/gitlab-org/prometheus-client-mmap/merge_requests/11 is ready - false + return @prometheus_metrics_enabled if defined?(@prometheus_metrics_enabled) + + @prometheus_metrics_enabled = prometheus_metrics_enabled_unmemoized end def registry -- cgit v1.2.1 From efe4cab92b1c93b2beb75fc6b4c0dbe0787d301e Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 23 Nov 2017 01:25:45 +0100 Subject: check method timing threshold when observing method performance --- lib/gitlab/metrics/method_call.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 5177d953795..03bdb5885a8 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -59,7 +59,7 @@ module Gitlab @cpu_time += cpu_time @call_count += 1 - if prometheus_enabled? + if prometheus_enabled? && above_threshold? self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0) end @@ -87,7 +87,7 @@ module Gitlab end def prometheus_enabled? - @prometheus_enabled ||= current_application_settings(:prometheus_metrics_method_instrumentation_enabled) + @prometheus_enabled ||= Gitlab::CurrentSettings.current_application_settings[:prometheus_metrics_method_instrumentation_enabled] end end end -- cgit v1.2.1 From 0051b5fbcc3154dacf20b1e89387b9ea55827266 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 23 Nov 2017 15:28:37 +0100 Subject: Use only real duration to measure method call performance via Prometheus --- lib/gitlab/metrics/method_call.rb | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 03bdb5885a8..518aefa19ee 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -6,29 +6,15 @@ module Gitlab BASE_LABELS = { module: nil, method: nil }.freeze attr_reader :real_time, :cpu_time, :call_count, :labels - def self.call_real_duration_histogram - return @call_real_duration_histogram if @call_real_duration_histogram - - MUTEX.synchronize do - @call_real_duration_histogram ||= Gitlab::Metrics.histogram( - :gitlab_method_call_real_duration_seconds, - 'Method calls real duration', - Transaction::BASE_LABELS.merge(BASE_LABELS), - [0.1, 0.2, 0.5, 1, 2, 5, 10] - ) - end - end - - def self.call_cpu_duration_histogram - return @call_cpu_duration_histogram if @call_cpu_duration_histogram + def self.call_duration_histogram + return @call_duration_histogram if @call_duration_histogram MUTEX.synchronize do @call_duration_histogram ||= Gitlab::Metrics.histogram( - :gitlab_method_call_cpu_duration_seconds, - 'Method calls cpu duration', + :gitlab_method_call_duration_seconds, + 'Method calls real duration', Transaction::BASE_LABELS.merge(BASE_LABELS), - [0.1, 0.2, 0.5, 1, 2, 5, 10] - ) + [0.01, 0.05, 0.1, 0.5, 1]) end end @@ -60,8 +46,7 @@ module Gitlab @call_count += 1 if prometheus_enabled? && above_threshold? - self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) - self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0) + self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) end retval -- cgit v1.2.1 From 46cd2d93bb23807b76bf20bb06e6ef93f1985ad9 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 23 Nov 2017 23:30:57 +0100 Subject: Use feature flag instead of application settigns to control if method calls should be instrumented --- lib/api/settings.rb | 3 --- lib/gitlab/metrics/method_call.rb | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/settings.rb b/lib/api/settings.rb index a43f8d0497c..851b226e9e5 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -66,9 +66,6 @@ module API optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' - given prometheus_metrics_enabled: ->(val) { val } do - requires :prometheus_metrics_method_instrumentation_enabled, type: Boolean, desc: 'Enable method call instrumentation' - end optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' given metrics_enabled: ->(val) { val } do requires :metrics_host, type: String, desc: 'The InfluxDB host' diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 518aefa19ee..65d55576ac2 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -45,7 +45,7 @@ module Gitlab @cpu_time += cpu_time @call_count += 1 - if prometheus_enabled? && above_threshold? + if call_measurement_enabled? && above_threshold? self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) end @@ -71,8 +71,8 @@ module Gitlab real_time >= Metrics.method_call_threshold end - def prometheus_enabled? - @prometheus_enabled ||= Gitlab::CurrentSettings.current_application_settings[:prometheus_metrics_method_instrumentation_enabled] + def call_measurement_enabled? + Feature.get(:prometheus_metrics_method_instrumentation).enabled? end end end -- cgit v1.2.1 From d6dd9d712ac24a095d0b0506731f9415b7c3b5f5 Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Fri, 24 Nov 2017 12:41:36 +0000 Subject: Fix ProtectedBranch access level validations Before an access_level was required in EE even when an it had been set for a user/group. --- lib/api/protected_branches.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 15fcb9e8e27..b5021e8a712 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -40,10 +40,10 @@ module API params do requires :name, type: String, desc: 'The name of the protected branch' optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER, - values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to push (defaults: `40`, master access level)' optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER, - values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to merge (defaults: `40`, master access level)' end post ':id/protected_branches' do -- cgit v1.2.1 From 8041a87288906e4b10b86a9a2ab9039036243a5d Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 24 Nov 2017 11:23:14 +0100 Subject: Drastically improve project search performance by no longer searching namespace name --- lib/gitlab/search_results.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index efe8095beea..fef9d3e31d4 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -30,7 +30,7 @@ module Gitlab def initialize(current_user, limit_projects, query) @current_user = current_user @limit_projects = limit_projects || Project.all - @query = Shellwords.shellescape(query) if query.present? + @query = query end def objects(scope, page = nil) -- cgit v1.2.1 From b2c5363da1bdfb4df8693de38f9d83fe203e6e99 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 24 Nov 2017 12:08:16 +0100 Subject: Rename to_fuzzy_arel to fuzzy_arel_match --- lib/gitlab/sql/pattern.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 7c2d1d8f887..8741aa0f1c4 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -19,7 +19,7 @@ module Gitlab query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end - def to_fuzzy_arel(column, query) + def fuzzy_arel_match(column, query) words = select_fuzzy_words(query) matches = words.map { |word| arel_table[column].matches(to_pattern(word)) } -- cgit v1.2.1 From 17069a9547c14c53491af76035c06dc66b8d8049 Mon Sep 17 00:00:00 2001 From: digitalMoksha Date: Fri, 24 Nov 2017 19:29:25 +0100 Subject: ignore hashed repositories when doing rake gitlab:cleanup:dirs --- lib/tasks/gitlab/cleanup.rake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 301affc9522..111576ea802 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,7 +5,8 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) + namespaces = Namespace.pluck(:path) + namespaces << '@hashed' # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| git_base_path = repository_storage['path'] all_dirs = Dir.glob(git_base_path + '/*') -- cgit v1.2.1 From 3bb103f28272bae4b541a3291d3ebd8396f97ca0 Mon Sep 17 00:00:00 2001 From: digitalMoksha Date: Sat, 25 Nov 2017 14:23:41 +0100 Subject: refactored the hashed repository name --- lib/tasks/gitlab/cleanup.rake | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 111576ea802..e3a18b327dd 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,12 +1,15 @@ namespace :gitlab do namespace :cleanup do + + HASHED_REPOSITORY_NAME = '@hashed' + desc "GitLab | Cleanup | Clean namespaces" task dirs: :environment do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] namespaces = Namespace.pluck(:path) - namespaces << '@hashed' # add so that it will be ignored + namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| git_base_path = repository_storage['path'] all_dirs = Dir.glob(git_base_path + '/*') @@ -63,7 +66,7 @@ namespace :gitlab do # TODO ignoring hashed repositories for now. But revisit to fully support # possible orphaned hashed repos - next if repo_with_namespace.start_with?('@hashed/') || Project.find_by_full_path(repo_with_namespace) + next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace) new_path = path + move_suffix puts path.inspect + ' -> ' + new_path.inspect -- cgit v1.2.1 From 30a9fc33f38ca552d5d53e7096edd2a1c1648db4 Mon Sep 17 00:00:00 2001 From: digitalMoksha Date: Sat, 25 Nov 2017 15:20:24 +0100 Subject: static-analysis fix --- lib/tasks/gitlab/cleanup.rake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index e3a18b327dd..eb0f757aea7 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,7 +1,6 @@ namespace :gitlab do namespace :cleanup do - - HASHED_REPOSITORY_NAME = '@hashed' + HASHED_REPOSITORY_NAME = '@hashed'.freeze desc "GitLab | Cleanup | Clean namespaces" task dirs: :environment do -- cgit v1.2.1 From 7fb1bb01bd669cc46514ed17b1b8822a1d962970 Mon Sep 17 00:00:00 2001 From: George Andrinopoulos Date: Sat, 25 Nov 2017 17:04:45 +0200 Subject: Create issue and merge request destroy services --- lib/api/issues.rb | 4 +++- lib/api/merge_requests.rb | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 74dfd9f96de..e60e00d7956 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -255,7 +255,9 @@ module API authorize!(:destroy_issue, issue) - destroy_conditionally!(issue) + destroy_conditionally!(issue) do |issue| + Issuable::DestroyService.new(user_project, current_user).execute(issue) + end end desc 'List merge requests closing issue' do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 5b4642a2f57..d34886fca2e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -167,7 +167,9 @@ module API authorize!(:destroy_merge_request, merge_request) - destroy_conditionally!(merge_request) + destroy_conditionally!(merge_request) do |merge_request| + Issuable::DestroyService.new(user_project, current_user).execute(merge_request) + end end params do -- cgit v1.2.1 From d4eea275310867eccc927d0e92a1d19a165f0668 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 24 Nov 2017 12:23:47 +0100 Subject: Modify fuzzy_arel_match to search for equality when a term shorter than 3 characters is provided --- lib/gitlab/sql/pattern.rb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 8741aa0f1c4..20ca36efb29 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -4,7 +4,7 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_WORD = /(?<=^| )"[^"]+"(?= |$)/ + REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/ class_methods do def to_pattern(query) @@ -20,11 +20,18 @@ module Gitlab end def fuzzy_arel_match(column, query) - words = select_fuzzy_words(query) + query = query.squish + return nil unless query.present? - matches = words.map { |word| arel_table[column].matches(to_pattern(word)) } + words = select_fuzzy_words(query) - matches.reduce { |result, match| result.and(match) } + if words.any? + words.map { |word| arel_table[column].matches(to_pattern(word)) }.reduce(:and) + else + # No words of at least 3 chars, but we can search for an exact + # case insensitive match with the query as a whole + arel_table[column].matches(sanitize_sql_like(query)) + end end def select_fuzzy_words(query) @@ -32,7 +39,7 @@ module Gitlab query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } - words = query.split(/\s+/) + words = query.split quoted_words.map! { |quoted_word| quoted_word[1..-2] } -- cgit v1.2.1 From da42dfb3cf4a2fb0cdcc1a3b41438516a0bed0e5 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 24 Nov 2017 12:24:24 +0100 Subject: Use fuzzy search with minimum length of 3 characters where appropriate --- lib/gitlab/sql/pattern.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 20ca36efb29..5f0c98cb5a4 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -7,6 +7,12 @@ module Gitlab REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/ class_methods do + def fuzzy_search(query, columns) + matches = columns.map { |col| fuzzy_arel_match(col, query) }.compact.reduce(:or) + + where(matches) + end + def to_pattern(query) if partial_matching?(query) "%#{sanitize_sql_like(query)}%" -- cgit v1.2.1 From 623eb68195d51ea50e09970771442d992ff19a4a Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 16 Nov 2017 17:12:33 +0100 Subject: Add new API endpoint - list jobs of a specified runner --- lib/api/runners.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'lib') diff --git a/lib/api/runners.rb b/lib/api/runners.rb index e816fcdd928..56b70681852 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -84,6 +84,18 @@ module API destroy_conditionally!(runner) end + + desc 'List jobs running on a runner' + params do + requires :id, type: Integer, desc: 'The ID of the runner' + use :pagination + end + get ':id/jobs' do + runner = get_runner(params[:id]) + authenticate_list_runners_jobs!(runner) + + present paginate(runner.builds.running), with: Entities::Job + end end params do @@ -192,6 +204,12 @@ module API forbidden!("No access granted") unless user_can_access_runner?(runner) end + def authenticate_list_runners_jobs!(runner) + return if current_user.admin? + + forbidden!("No access granted") unless user_can_access_runner?(runner) + end + def user_can_access_runner?(runner) current_user.ci_authorized_runners.exists?(runner.id) end -- cgit v1.2.1 From 8d3e80692cbeea06dd28a052554f0c262004e18d Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 16 Nov 2017 19:44:14 +0100 Subject: Add information about project --- lib/api/entities.rb | 17 +++++++++++++---- lib/api/runners.rb | 6 ++++-- 2 files changed, 17 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 7d5d68c8f14..cea9e9a8028 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -80,16 +80,21 @@ module API expose :group_access, as: :group_access_level end - class BasicProjectDetails < Grape::Entity - expose :id, :description, :default_branch, :tag_list - expose :ssh_url_to_repo, :http_url_to_repo, :web_url + class ProjectIdentity < Grape::Entity + expose :id, :description expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :created_at + end + + class BasicProjectDetails < ProjectIdentity + expose :default_branch, :tag_list + expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :avatar_url do |project, options| project.avatar_url(only_path: false) end expose :star_count, :forks_count - expose :created_at, :last_activity_at + expose :last_activity_at end class Project < BasicProjectDetails @@ -838,6 +843,10 @@ module API expose :pipeline, with: PipelineBasic end + class JobWithProject < Job + expose :project, with: ProjectIdentity + end + class Trigger < Grape::Entity expose :id expose :token, :description diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 56b70681852..b92a2c36cf3 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -85,7 +85,9 @@ module API destroy_conditionally!(runner) end - desc 'List jobs running on a runner' + desc 'List jobs running on a runner' do + success Entities::JobWithProject + end params do requires :id, type: Integer, desc: 'The ID of the runner' use :pagination @@ -94,7 +96,7 @@ module API runner = get_runner(params[:id]) authenticate_list_runners_jobs!(runner) - present paginate(runner.builds.running), with: Entities::Job + present paginate(runner.builds.running), with: Entities::JobWithProject end end -- cgit v1.2.1 From b7ed102ea601fb4c6f65c5a982058f8c92883d31 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 21 Nov 2017 12:37:07 +0100 Subject: Allow filtering by 'status' --- lib/api/runners.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/runners.rb b/lib/api/runners.rb index b92a2c36cf3..18f9f142580 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -90,13 +90,20 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the runner' + optional :status, type: String, desc: 'Status of job' use :pagination end get ':id/jobs' do runner = get_runner(params[:id]) authenticate_list_runners_jobs!(runner) - present paginate(runner.builds.running), with: Entities::JobWithProject + jobs = runner.builds + if params[:status] + not_found!('Status') unless Ci::Build::AVAILABLE_STATUSES.include?(params[:status]) + jobs = jobs.where(status: params[:status].to_sym) + end + + present paginate(jobs), with: Entities::JobWithProject end end -- cgit v1.2.1 From 7b643c02c20e7552a35c09adf8d8dca1e8c3a4a3 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 21 Nov 2017 12:37:18 +0100 Subject: Modify output --- lib/api/entities.rb | 11 +++++++---- lib/api/runners.rb | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cea9e9a8028..ce332fe85d2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -832,18 +832,21 @@ module API expose :id, :sha, :ref, :status end - class Job < Grape::Entity + class JobBasic < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at expose :duration expose :user, with: User - expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :commit, with: Commit - expose :runner, with: Runner expose :pipeline, with: PipelineBasic end - class JobWithProject < Job + class Job < JobBasic + expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } + expose :runner, with: Runner + end + + class JobBasicWithProject < JobBasic expose :project, with: ProjectIdentity end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 18f9f142580..3ea7340d9be 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -86,7 +86,7 @@ module API end desc 'List jobs running on a runner' do - success Entities::JobWithProject + success Entities::JobBasicWithProject end params do requires :id, type: Integer, desc: 'The ID of the runner' @@ -103,7 +103,7 @@ module API jobs = jobs.where(status: params[:status].to_sym) end - present paginate(jobs), with: Entities::JobWithProject + present paginate(jobs), with: Entities::JobBasicWithProject end end -- cgit v1.2.1 From 13a902a9a42c0909ad8c6790c040447a9e12211f Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 27 Nov 2017 22:59:01 +0100 Subject: Refactorize jobs finding logic --- lib/api/runners.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 3ea7340d9be..996457c5dfe 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -90,18 +90,14 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the runner' - optional :status, type: String, desc: 'Status of job' + optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES use :pagination end get ':id/jobs' do runner = get_runner(params[:id]) authenticate_list_runners_jobs!(runner) - jobs = runner.builds - if params[:status] - not_found!('Status') unless Ci::Build::AVAILABLE_STATUSES.include?(params[:status]) - jobs = jobs.where(status: params[:status].to_sym) - end + jobs = RunnerJobsFinder.new(runner, params).execute present paginate(jobs), with: Entities::JobBasicWithProject end -- cgit v1.2.1 From 53da3d976f3705a87edc50dca41748b5e479fc83 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 27 Nov 2017 22:35:16 +0900 Subject: Replce kubernetes_service and deployment_service to deployment_platform --- lib/gitlab/prometheus/queries/query_additional_metrics.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 7ac6162b54d..5cddc96a643 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -76,7 +76,7 @@ module Gitlab timeframe_start: timeframe_start, timeframe_end: timeframe_end, ci_environment_slug: environment.slug, - kube_namespace: environment.project.kubernetes_service&.actual_namespace || '', + kube_namespace: environment.project.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} } end -- cgit v1.2.1 From 60ac7e0cf3ea6e214d89ca10d4011cde69284ca0 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 7 Nov 2017 15:13:48 +0100 Subject: Add controller#action metrics on Gitaly At this time we had good metrics on what number or requests each GRPC received, but were in the dark what controller#action combination was responsable. Or if Sidekiq was responsable. Now added are call counts per service and rpc matched with controller#action combinations. --- lib/gitlab/gitaly_client.rb | 49 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 572f4c892f6..d7375938ab6 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -31,14 +31,38 @@ module Gitlab CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new - private_constant :MUTEX + METRICS_MUTEX = Mutex.new + private_constant :MUTEX, :METRICS_MUTEX class << self - attr_accessor :query_time, :migrate_histogram + attr_accessor :query_time end self.query_time = 0 - self.migrate_histogram = Gitlab::Metrics.histogram(:gitaly_migrate_call_duration, "Gitaly migration call execution timings") + + def self.migrate_histogram + @migrate_histogram ||= + METRICS_MUTEX.synchronize do + # If a thread was blocked on the mutex, the value was set already + return @migrate_histogram if @migrate_histogram + + Gitlab::Metrics.histogram(:gitaly_migrate_call_duration_seconds, + "Gitaly migration call execution timings", + gitaly_enabled: nil, feature: nil) + end + end + + def self.gitaly_call_histogram + @gitaly_call_histogram ||= + METRICS_MUTEX.synchronize do + # If a thread was blocked on the mutex, the value was set already + return @gitaly_call_histogram if @gitaly_call_histogram + + Gitlab::Metrics.histogram(:gitaly_controller_action_duration_seconds, + "Gitaly endpoint histogram by controller and action combination", + Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil)) + end + end def self.stub(name, storage) MUTEX.synchronize do @@ -94,7 +118,7 @@ module Gitlab # end # def self.call(storage, service, rpc, request, remote_storage: nil) - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + start = Gitlab::Metrics::System.monotonic_time enforce_gitaly_request_limits(:call) kwargs = request_kwargs(storage, remote_storage: remote_storage) @@ -102,8 +126,19 @@ module Gitlab stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend ensure - self.query_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + duration = Gitlab::Metrics::System.monotonic_time - start + + # Keep track, seperately, for the performance bar + self.query_time += duration + gitaly_call_histogram.observe( + current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), + duration) + end + + def self.current_transaction_labels + Gitlab::Metrics::Transaction.current&.labels || {} end + private_class_method :current_transaction_labels def self.request_kwargs(storage, remote_storage: nil) encoded_token = Base64.strict_encode64(token(storage).to_s) @@ -178,10 +213,10 @@ module Gitlab feature_stack = Thread.current[:gitaly_feature_stack] ||= [] feature_stack.unshift(feature) begin - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + start = Gitlab::Metrics::System.monotonic_time yield is_enabled ensure - total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + total_time = Gitlab::Metrics::System.monotonic_time - start migrate_histogram.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) feature_stack.shift Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? -- cgit v1.2.1 From 4ebbfe5d3e0dbe06346ee0c64a8f62ec11f9b6e8 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Tue, 21 Nov 2017 16:58:08 +0000 Subject: Remove serialised diff and commit columns The st_commits and st_diffs columns on merge_request_diffs historically held the YAML-serialised data for a merge request diff, in a variety of formats. Since 9.5, these have been migrated in the background to two new tables: merge_request_diff_commits and merge_request_diff_files. That has the advantage that we can actually query the data (for instance, to find out how many commits we've stored), and that it can't be in a variety of formats, but must match the new schema. This is the final step of that journey, where we drop those columns and remove all references to them. This is a breaking change to the importer, because we can no longer import diffs created in the old format, and we cannot guarantee the export will be in the new format unless it was generated after this commit. --- lib/gitlab/cycle_analytics/plan_event_fetcher.rb | 8 +------- lib/gitlab/import_export.rb | 2 +- lib/gitlab/import_export/import_export.yml | 2 -- lib/gitlab/import_export/relation_factory.rb | 8 -------- 4 files changed, 2 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 2479b4a7706..9230894877f 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -3,7 +3,6 @@ module Gitlab class PlanEventFetcher < BaseEventFetcher def initialize(*args) @projections = [mr_diff_table[:id], - mr_diff_table[:st_commits], issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) @@ -37,12 +36,7 @@ module Gitlab def first_time_reference_commit(event) return nil unless event && merge_request_diff_commits - commits = - if event['st_commits'].present? - YAML.load(event['st_commits']) - else - merge_request_diff_commits[event['id'].to_i] - end + commits = merge_request_diff_commits[event['id'].to_i] return nil if commits.blank? diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 50ee879129c..2066005dddc 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.0'.freeze + VERSION = '0.2.1'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 263599831bf..f2b193c79cb 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -133,8 +133,6 @@ methods: - :type services: - :type - merge_request_diff: - - :utf8_st_diffs merge_request_diff_files: - :utf8_diff merge_requests: diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 2b34ceb5831..d7d1b05e8b9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -58,7 +58,6 @@ module Gitlab def setup_models case @relation_name - when :merge_request_diff then setup_st_diff_commits when :merge_request_diff_files then setup_diff when :notes then setup_note when :project_label, :project_labels then setup_label @@ -208,13 +207,6 @@ module Gitlab relation_class: relation_class) end - def setup_st_diff_commits - @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') - - HashUtil.deep_symbolize_array!(@relation_hash['st_diffs']) - HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) - end - def setup_diff @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end -- cgit v1.2.1 From 3c6a4d63636ba41dad0ce63cf536761fc3b5ef64 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 24 Nov 2017 11:58:05 +0000 Subject: Ensure MRs always use branch refs for comparison If a merge request was created with a branch name that also matched a tag name, we'd generate a comparison to or from the tag respectively, rather than the branch. Merging would still use the branch, of course. To avoid this, ensure that when we get the branch heads, we prepend the reference prefix for branches, which will ensure that we generate the correct comparison. --- lib/gitlab/git/repository.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index a6e7c410bdd..d399636bb28 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1046,9 +1046,15 @@ module Gitlab end def with_repo_tmp_commit(start_repository, start_branch_name, sha) + source_ref = start_branch_name + + unless Gitlab::Git.branch_ref?(source_ref) + source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" + end + tmp_ref = fetch_ref( start_repository, - source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + source_ref: source_ref, target_ref: "refs/tmp/#{SecureRandom.hex}" ) -- cgit v1.2.1 From a402056472aa9d8472b9ef92e70128e472b2b030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Tue, 28 Nov 2017 11:17:57 +0100 Subject: SSHUploadPack over Gitaly is now OptOut - Better gitaly-handling in /api/internal/allowed specs --- lib/api/helpers/internal_helpers.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 4b3c473b0bb..d6dea4c30e3 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -2,8 +2,8 @@ module API module Helpers module InternalHelpers SSH_GITALY_FEATURES = { - 'git-receive-pack' => :ssh_receive_pack, - 'git-upload-pack' => :ssh_upload_pack + 'git-receive-pack' => [:ssh_receive_pack, Gitlab::GitalyClient::MigrationStatus::OPT_IN], + 'git-upload-pack' => [:ssh_upload_pack, Gitlab::GitalyClient::MigrationStatus::OPT_OUT] }.freeze def wiki? @@ -102,8 +102,8 @@ module API # Return the Gitaly Address if it is enabled def gitaly_payload(action) - feature = SSH_GITALY_FEATURES[action] - return unless feature && Gitlab::GitalyClient.feature_enabled?(feature) + feature, status = SSH_GITALY_FEATURES[action] + return unless feature && Gitlab::GitalyClient.feature_enabled?(feature, status: status) { repository: repository.gitaly_repository, -- cgit v1.2.1 From 64e5f996fa01d2e9c8462cfbb647997531eaf6c7 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 29 Nov 2017 09:12:12 +0000 Subject: Add timeouts for Gitaly calls --- lib/api/settings.rb | 3 ++ lib/gitlab/gitaly_client.rb | 43 +++++++++++++++++++++++--- lib/gitlab/gitaly_client/commit_service.rb | 30 +++++++++--------- lib/gitlab/gitaly_client/ref_service.rb | 3 +- lib/gitlab/gitaly_client/repository_service.rb | 9 ++++-- 5 files changed, 65 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 06373fe5069..cee4d309816 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -123,6 +123,9 @@ module API end optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index d7375938ab6..f27cd800bdd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -117,11 +117,11 @@ module Gitlab # kwargs.merge(deadline: Time.now + 10) # end # - def self.call(storage, service, rpc, request, remote_storage: nil) + def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil) start = Gitlab::Metrics::System.monotonic_time enforce_gitaly_request_limits(:call) - kwargs = request_kwargs(storage, remote_storage: remote_storage) + kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend @@ -140,7 +140,7 @@ module Gitlab end private_class_method :current_transaction_labels - def self.request_kwargs(storage, remote_storage: nil) + def self.request_kwargs(storage, timeout, remote_storage: nil) encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { 'authorization' => "Bearer #{encoded_token}", @@ -152,7 +152,22 @@ module Gitlab metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage - { metadata: metadata } + result = { metadata: metadata } + + # nil timeout indicates that we should use the default + timeout = default_timeout if timeout.nil? + + return result unless timeout > 0 + + # Do not use `Time.now` for deadline calculation, since it + # will be affected by Timecop in some tests, but grpc's c-core + # uses system time instead of timecop's time, so tests will fail + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will + # circumvent timecop + deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + result[:deadline] = deadline + + result end def self.token(storage) @@ -325,6 +340,26 @@ module Gitlab Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) end + # The default timeout on all Gitaly calls + def self.default_timeout + return 0 if Sidekiq.server? + + timeout(:gitaly_timeout_default) + end + + def self.fast_timeout + timeout(:gitaly_timeout_fast) + end + + def self.medium_timeout + timeout(:gitaly_timeout_medium) + end + + def self.timeout(timeout_name) + Gitlab::CurrentSettings.current_application_settings[timeout_name] + end + private_class_method :timeout + # Count a stack. Used for n+1 detection def self.count_stack return unless RequestStore.active? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index da5505cb2fe..34807d280e5 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -16,7 +16,7 @@ module Gitlab revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request) + response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map { |d| EncodingHelper.encode!(d.dup) } end @@ -29,7 +29,7 @@ module Gitlab child_id: child_id ) - GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value + GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value end def diff(from, to, options = {}) @@ -77,7 +77,7 @@ module Gitlab limit: limit.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) + response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) entry = nil data = '' @@ -102,7 +102,7 @@ module Gitlab path: path.present? ? GitalyClient.encode(path) : '.' ) - response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) + response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) response.flat_map do |message| message.entries.map do |gitaly_tree_entry| @@ -129,7 +129,7 @@ module Gitlab request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? request.path = options[:path] if options[:path].present? - GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count + GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end def last_commit_for_path(revision, path) @@ -139,7 +139,7 @@ module Gitlab path: GitalyClient.encode(path.to_s) ) - gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit + gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit return unless gitaly_commit Gitlab::Git::Commit.new(@repository, gitaly_commit) @@ -152,7 +152,7 @@ module Gitlab to: to ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request) + response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -165,7 +165,7 @@ module Gitlab ) request.order = opts[:order].upcase if opts[:order].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -179,7 +179,7 @@ module Gitlab offset: offset.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request) + response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -197,7 +197,7 @@ module Gitlab path: GitalyClient.encode(path) ) - response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request) + response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce("") { |memo, msg| memo << msg.data } end @@ -207,7 +207,7 @@ module Gitlab revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) response.commit end @@ -217,7 +217,7 @@ module Gitlab repository: @gitaly_repo, revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout) response.sum(&:data) end @@ -227,7 +227,7 @@ module Gitlab repository: @gitaly_repo, revision: GitalyClient.encode(revision) ) - GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) + GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end def find_commits(options) @@ -245,7 +245,7 @@ module Gitlab request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -259,7 +259,7 @@ module Gitlab request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) GitalyClient::DiffStitcher.new(response) end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 31b04bc2650..066e4e183c0 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -46,7 +46,8 @@ module Gitlab commit_id: commit_id, prefix: ref_prefix ) - encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup) + response = GitalyClient.call(@storage, :ref_service, :find_ref_name, request, timeout: GitalyClient.medium_timeout) + encode!(response.name.dup) end def count_tag_names diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 70cb16bd810..b9e606592d7 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -10,7 +10,9 @@ module Gitlab def exists? request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repository_exists, request).exists + response = GitalyClient.call(@storage, :repository_service, :repository_exists, request, timeout: GitalyClient.fast_timeout) + + response.exists end def garbage_collect(create_bitmap) @@ -30,7 +32,8 @@ module Gitlab def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repository_size, request).size + response = GitalyClient.call(@storage, :repository_service, :repository_size, request) + response.size end def apply_gitattributes(revision) @@ -61,7 +64,7 @@ module Gitlab def has_local_branches? request = Gitaly::HasLocalBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request) + response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request, timeout: GitalyClient.fast_timeout) response.value end -- cgit v1.2.1 From 4c4109e1b942dae17f3a716f51a0da15cd335043 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 24 Nov 2017 15:24:38 +0100 Subject: Create fork networks for forks for which the source was deleted. That way we can join forks-of-forks into the same network even if their original source was deleted. Consider the following: `awesome-oss/badger` is forked to `coolstuff/badger`, which is forked to `user-a/badger` which in turn is forked to `user-b/badger`. When `awesome-oss/badger` is deleted, we will now create a fork network with `coolstuff/badger` as the root. The `user-a/badger` and `user-b/badger` projects will be added to the network. --- .../populate_fork_networks_range.rb | 78 ++++++++++++++++++++-- 1 file changed, 71 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb index f8508b5fbdf..a976cb4c243 100644 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -1,30 +1,55 @@ # frozen_string_literal: true -# rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/LineLength -# rubocop:disable Style/Documentation module Gitlab module BackgroundMigration + # This background migration is going to create all `fork_networks` and + # the `fork_network_members` for the roots of fork networks based on the + # existing `forked_project_links`. + # + # When the source of a fork is deleted, we will create the fork with the + # target project as the root. This way, when there are forks of the target + # project, they will be joined into the same fork network. + # + # When the `fork_networks` and memberships for the root projects are created + # the `CreateForkNetworkMembershipsRange` migration is scheduled. This + # migration will create the memberships for all remaining forks-of-forks class PopulateForkNetworksRange def perform(start_id, end_id) - log("Creating fork networks for forked project links: #{start_id} - #{end_id}") + create_fork_networks_for_existing_projects(start_id, end_id) + create_fork_networks_for_missing_projects(start_id, end_id) + create_fork_networks_memberships_for_root_projects(start_id, end_id) + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength + BackgroundMigrationWorker.perform_in( + delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] + ) + end + + def create_fork_networks_for_existing_projects(start_id, end_id) + log("Creating fork networks: #{start_id} - #{end_id}") ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS INSERT INTO fork_networks (root_project_id) SELECT DISTINCT forked_project_links.forked_from_project_id FROM forked_project_links + -- Exclude the forks that are not the first level fork of a project WHERE NOT EXISTS ( SELECT true FROM forked_project_links inner_links WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id ) + + /* Exclude the ones that are already created, in case the fork network + was already created for another fork of the project. + */ AND NOT EXISTS ( SELECT true FROM fork_networks WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id ) + + -- Only create a fork network for a root project that still exists AND EXISTS ( SELECT true FROM projects @@ -32,7 +57,45 @@ module Gitlab ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} INSERT_NETWORKS + end + + def create_fork_networks_for_missing_projects(start_id, end_id) + log("Creating fork networks with missing root: #{start_id} - #{end_id}") + ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS + INSERT INTO fork_networks (root_project_id) + SELECT DISTINCT forked_project_links.forked_to_project_id + + FROM forked_project_links + + -- Exclude forks that are not the root forks + WHERE NOT EXISTS ( + SELECT true + FROM forked_project_links inner_links + WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id + ) + /* Exclude the ones that are already created, in case this migration is + re-run + */ + AND NOT EXISTS ( + SELECT true + FROM fork_networks + WHERE forked_project_links.forked_to_project_id = fork_networks.root_project_id + ) + + /* Exclude projects for which the project still exists, those are + Processed in the previous step of this migration + */ + AND NOT EXISTS ( + SELECT true + FROM projects + WHERE projects.id = forked_project_links.forked_from_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_NETWORKS + end + + def create_fork_networks_memberships_for_root_projects(start_id, end_id) log("Creating memberships for root projects: #{start_id} - #{end_id}") ActiveRecord::Base.connection.execute <<~INSERT_ROOT @@ -41,8 +104,12 @@ module Gitlab FROM fork_networks + /* Joining both on forked_from- and forked_to- so we could create the + memberships for forks for which the source was deleted + */ INNER JOIN forked_project_links ON forked_project_links.forked_from_project_id = fork_networks.root_project_id + OR forked_project_links.forked_to_project_id = fork_networks.root_project_id WHERE NOT EXISTS ( SELECT true @@ -51,9 +118,6 @@ module Gitlab ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} INSERT_ROOT - - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY - BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]) end def log(message) -- cgit v1.2.1 From 57d9121127eb9745ea196bbd8596ffa03afdee68 Mon Sep 17 00:00:00 2001 From: haseeb Date: Wed, 29 Nov 2017 16:22:22 +0000 Subject: support ordering of project notes in notes api --- lib/api/notes.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/notes.rb b/lib/api/notes.rb index ceaaeca4046..3588dc85c9e 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -18,6 +18,10 @@ module API end params do requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return notes ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return notes sorted in `asc` or `desc` order.' use :pagination end get ":id/#{noteables_str}/:noteable_id/notes" do @@ -29,11 +33,12 @@ module API # at the DB query level (which we cannot in that case), the current # page can have less elements than :per_page even if # there's more than one page. + raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort]) notes = # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(noteable.notes.with_metadata) + paginate(raw_notes) .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note else -- cgit v1.2.1 From 4925ec50877c9bf1338d41a7676b3644f18370f7 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 28 Nov 2017 21:43:31 +0800 Subject: We could simply count the commits --- lib/gitlab/git/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d399636bb28..fb9c3e92d3f 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -505,7 +505,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - Commit.between(self, from, to).size + count_commits(ref: "#{from}..#{to}") end # Returns the SHA of the most recent common ancestor of +from+ and +to+ -- cgit v1.2.1 From 869877ab2691f94927e49610349e284633688819 Mon Sep 17 00:00:00 2001 From: haseeb Date: Thu, 30 Nov 2017 09:57:58 +0000 Subject: fix for special charecter in file names --- lib/gitlab/project_search_results.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 561aa9e162c..e2662fc362b 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -47,8 +47,11 @@ module Gitlab startline = 0 result.each_line.each_with_index do |line, index| - if line =~ /^.*:.*:\d+:/ - ref, filename, startline = line.split(':') + matches = line.match(/^(?[^:]*):(?.*):(?\d+):/) + if matches + ref = matches[:ref] + filename = matches[:filename] + startline = matches[:startline] startline = startline.to_i - index extname = Regexp.escape(File.extname(filename)) basename = filename.sub(/#{extname}$/, '') -- cgit v1.2.1 From 60056d67a08ee0afd211e64a2a364371f719cd07 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 30 Nov 2017 17:05:55 +0100 Subject: Add link to gitaly converation This endpoint still has to be migrated, and this comment makes sure everyone knows we are aware of this one. [ci skip] --- lib/gitlab/git/blob.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index ddd52136bc4..228d97a87ab 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -49,6 +49,7 @@ module Gitlab # Keep in mind that this method may allocate a lot of memory. It is up # to the caller to limit the number of blobs and blob_size_limit. # + # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 def batch(repository, blob_references, blob_size_limit: nil) blob_size_limit ||= MAX_DATA_DISPLAY_SIZE blob_references.map do |sha, path| -- cgit v1.2.1 From 66127221fea44fd0dcf35b2ff592f5efe21c530f Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 29 Nov 2017 00:03:43 -0500 Subject: Gracefully handle case when repository's root ref does not exist This was failing regularly with an Error 500 when the API branches endpoint was used. Closes #40615 --- lib/gitlab/git/repository.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d399636bb28..4657e2cc6ae 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1253,7 +1253,11 @@ module Gitlab # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 def git_merged_branch_names(branch_names = []) - root_sha = find_branch(root_ref).target + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha git_arguments = %W[branch --merged #{root_sha} -- cgit v1.2.1 From 97552d46fef3958aad4dfa3b67d43edb78513f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 1 Dec 2017 15:55:06 +0100 Subject: Ensure `Namespace`'s is namespaced in `Gitlab::Kubernetes::Helm#initialize` and fix a transient failing spec due to that MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/kubernetes/helm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 7a50f07f3c5..407cdefc04d 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -18,7 +18,7 @@ module Gitlab def initialize(kubeclient) @kubeclient = kubeclient - @namespace = Namespace.new(NAMESPACE, kubeclient) + @namespace = Gitlab::Kubernetes::Namespace.new(NAMESPACE, kubeclient) end def install(command) -- cgit v1.2.1 From b72d243872fae75f6751b9a16b9668acf595894b Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 27 Nov 2017 10:44:36 +0100 Subject: Keep track of all storage keys in a set That way we don't need the to scan the entire keyspace to get all known keys --- lib/gitlab/git/storage.rb | 1 + lib/gitlab/git/storage/circuit_breaker.rb | 14 +++++++++----- lib/gitlab/git/storage/health.rb | 25 +++++++------------------ 3 files changed, 17 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb index 99518c9b1e4..5933312b0b5 100644 --- a/lib/gitlab/git/storage.rb +++ b/lib/gitlab/git/storage.rb @@ -15,6 +15,7 @@ module Gitlab Failing = Class.new(Inaccessible) REDIS_KEY_PREFIX = 'storage_accessible:'.freeze + REDIS_KNOWN_KEYS = "#{REDIS_KEY_PREFIX}known_keys_set".freeze def self.redis Gitlab::Redis::SharedState diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index be7598ef011..4328c0ea29b 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -13,10 +13,8 @@ module Gitlab delegate :last_failure, :failure_count, to: :failure_info def self.reset_all! - pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*" - Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.scan_each(match: pattern).to_a + all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) redis.del(*all_storage_keys) unless all_storage_keys.empty? end @@ -135,23 +133,29 @@ module Gitlab redis.hset(cache_key, :last_failure, last_failure.to_i) redis.hincrby(cache_key, :failure_count, 1) redis.expire(cache_key, failure_reset_time) + maintain_known_keys(redis) end end end def track_storage_accessible - return if no_failures? - @failure_info = FailureInfo.new(nil, 0) Gitlab::Git::Storage.redis.with do |redis| redis.pipelined do redis.hset(cache_key, :last_failure, nil) redis.hset(cache_key, :failure_count, 0) + maintain_known_keys(redis) end end end + def maintain_known_keys(redis) + expire_time = Time.now.to_i + failure_reset_time + redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) + redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) + end + def get_failure_info last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| redis.hmget(cache_key, :last_failure, :failure_count) diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 7049772fe3b..90bbe85fd37 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -4,8 +4,8 @@ module Gitlab class Health attr_reader :storage_name, :info - def self.pattern_for_storage(storage_name) - "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:*" + def self.prefix_for_storage(storage_name) + "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:" end def self.for_all_storages @@ -25,26 +25,15 @@ module Gitlab private_class_method def self.all_keys_for_storages(storage_names, redis) keys_per_storage = {} + all_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - redis.pipelined do - storage_names.each do |storage_name| - pattern = pattern_for_storage(storage_name) - matched_keys = redis.scan_each(match: pattern) + storage_names.each do |storage_name| + prefix = prefix_for_storage(storage_name) - keys_per_storage[storage_name] = matched_keys - end + keys_per_storage[storage_name] = all_keys.select { |key| key.starts_with?(prefix) } end - # We need to make sure each lazy-loaded `Enumerator` for matched keys - # is loaded into an array. - # - # Otherwise it would be loaded in the second `Redis#pipelined` block - # within `.load_for_keys`. In this pipelined call, the active - # Redis-client changes again, so the values would not be available - # until the end of that pipelined-block. - keys_per_storage.each do |storage_name, key_future| - keys_per_storage[storage_name] = key_future.to_a - end + keys_per_storage end private_class_method def self.load_for_keys(keys_per_storage, redis) -- cgit v1.2.1 From 88e3ce30ae97ac8eb4b25381cfbe7772819cce0c Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 19 Nov 2017 06:35:25 -0800 Subject: Optimize API /groups/:id/projects by preloading associations Closes #40308 --- lib/api/groups.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index bcf2e6dae1d..7e9a5502949 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -172,6 +172,7 @@ module API get ":id/projects" do group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = projects.preload(:fork_network, :forked_project_link, :project_feature, :project_group_links, :tags, :taggings, :group, :namespace) projects = reorder_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user -- cgit v1.2.1 From 39d293abd2ec135c53448552d482d17f9726701c Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 19 Nov 2017 15:25:48 -0800 Subject: Omit the `all` call after Project#project_group_links to avoid unnecessary loads --- lib/api/entities.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index ce332fe85d2..ca7708e366e 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -156,7 +156,7 @@ module API expose :public_builds, as: :public_jobs expose :ci_config_path expose :shared_with_groups do |project, options| - SharedGroup.represent(project.project_group_links.all, options) + SharedGroup.represent(project.project_group_links, options) end expose :only_allow_merge_if_pipeline_succeeds expose :request_access_enabled -- cgit v1.2.1 From 02cd1702b27ddb15f15f7efeb5ce60412492e41d Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 19 Nov 2017 16:02:46 -0800 Subject: Only serialize namespace essentials in group's projects API response --- lib/api/entities.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index ca7708e366e..5b9b0afed0f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -146,7 +146,7 @@ module API expose :shared_runners_enabled expose :lfs_enabled?, as: :lfs_enabled expose :creator_id - expose :namespace, using: 'API::Entities::Namespace' + expose :namespace, using: 'API::Entities::NamespaceBasic' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -618,9 +618,11 @@ module API expose :created_at end - class Namespace < Grape::Entity + class NamespaceBasic < Grape::Entity expose :id, :name, :path, :kind, :full_path, :parent_id + end + class Namespace < NamespaceBasic expose :members_count_with_descendants, if: -> (namespace, opts) { expose_members_count_with_descendants?(namespace, opts) } do |namespace, _| namespace.users_with_descendants.count end -- cgit v1.2.1 From 0dfb8801012de19be006026a01d7db0f04a2d3b3 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 19 Nov 2017 16:46:40 -0800 Subject: Preload project route to avoid N+1 query --- lib/api/groups.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 7e9a5502949..df2d97bceda 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -172,7 +172,7 @@ module API get ":id/projects" do group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute - projects = projects.preload(:fork_network, :forked_project_link, :project_feature, :project_group_links, :tags, :taggings, :group, :namespace) + projects = projects.preload(:fork_network, :forked_project_link, :project_feature, :project_group_links, :tags, :taggings, :group, :namespace, :route) projects = reorder_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user -- cgit v1.2.1 From a2babf32fe2b57850bd678919947d53e5127f6fe Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 24 Nov 2017 13:20:52 +0100 Subject: More preloading improvement and added batch count services --- lib/api/groups.rb | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index df2d97bceda..746dcc9fc91 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -52,6 +52,24 @@ module API groups end + def find_group_projects(params) + group = find_group!(params[:id]) + projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = projects.preload(:project_feature, :route, :group) + .preload(namespace: [:route, :owner], + tags: :taggings, + project_group_links: :group, + fork_network: :root_project, + forked_project_link: :forked_from_project, + forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) + projects = reorder_projects(projects) + paginated_projects = paginate(projects) + projects_with_fork = paginated_projects + paginated_projects.map(&:forked_from_project).compact + ::Projects::BatchForksCountService.new(projects_with_fork).refresh_cache + ::Projects::BatchOpenIssuesCountService.new(paginated_projects).refresh_cache + paginated_projects + end + def present_groups(params, groups) options = { with: Entities::Group, @@ -170,12 +188,9 @@ module API use :pagination end get ":id/projects" do - group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute - projects = projects.preload(:fork_network, :forked_project_link, :project_feature, :project_group_links, :tags, :taggings, :group, :namespace, :route) - projects = reorder_projects(projects) + projects = find_group_projects(params) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project - present paginate(projects), with: entity, current_user: current_user + present projects, with: entity, current_user: current_user end desc 'Get a list of subgroups in this group.' do -- cgit v1.2.1 From 7c7877b54dfb0a07bf128102226e338463654431 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 24 Nov 2017 13:32:01 +0100 Subject: Small renaming --- lib/api/groups.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 746dcc9fc91..05443329a32 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -63,11 +63,11 @@ module API forked_project_link: :forked_from_project, forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) projects = reorder_projects(projects) - paginated_projects = paginate(projects) - projects_with_fork = paginated_projects + paginated_projects.map(&:forked_from_project).compact + projects = paginate(projects) + projects_with_fork = projects + projects.map(&:forked_from_project).compact ::Projects::BatchForksCountService.new(projects_with_fork).refresh_cache - ::Projects::BatchOpenIssuesCountService.new(paginated_projects).refresh_cache - paginated_projects + ::Projects::BatchOpenIssuesCountService.new(projects).refresh_cache + projects end def present_groups(params, groups) -- cgit v1.2.1 From c85f9c5b1d320f7a6f75e3d08bbafd2fb20d3f58 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 24 Nov 2017 17:37:02 +0100 Subject: Code review comments applied --- lib/api/groups.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 05443329a32..83c7898150a 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -64,8 +64,8 @@ module API forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) projects = reorder_projects(projects) projects = paginate(projects) - projects_with_fork = projects + projects.map(&:forked_from_project).compact - ::Projects::BatchForksCountService.new(projects_with_fork).refresh_cache + projects_with_forks = projects + projects.map(&:forked_from_project).compact + ::Projects::BatchForksCountService.new(projects_with_forks).refresh_cache ::Projects::BatchOpenIssuesCountService.new(projects).refresh_cache projects end -- cgit v1.2.1 From 58c5b463ff75618a557d067c16f49ef581cda85c Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Mon, 27 Nov 2017 12:24:33 +0100 Subject: Refactored /projects and /user/:user_id/projects endpoints --- lib/api/entities.rb | 48 ++++++++++++++++++++++++++++-------- lib/api/groups.rb | 16 +++--------- lib/api/projects.rb | 4 +-- lib/api/projects_relation_builder.rb | 34 +++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 25 deletions(-) create mode 100644 lib/api/projects_relation_builder.rb (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5b9b0afed0f..8f41b930c0a 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -94,7 +94,13 @@ module API project.avatar_url(only_path: false) end expose :star_count, :forks_count - expose :last_activity_at + expose :created_at, :last_activity_at + + def self.preload_relation(projects_relation) + projects_relation.preload(:project_feature, :route) + .preload(namespace: [:route, :owner], + tags: :taggings) + end end class Project < BasicProjectDetails @@ -128,6 +134,18 @@ module API expose :members do |project| expose_url(api_v4_projects_members_path(id: project.id)) end + + def self.preload_relation(projects_relation) + super(projects_relation).preload(:group) + .preload(project_group_links: :group, + fork_network: :root_project, + forked_project_link: :forked_from_project, + forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) + end + + def self.forks_counting_projects(projects_relation) + projects_relation + projects_relation.map(&:forked_from_project).compact + end end expose :archived?, as: :archived @@ -635,9 +653,16 @@ module API class MemberAccess < Grape::Entity expose :access_level expose :notification_level do |member, options| - if member.notification_setting - ::NotificationSetting.levels[member.notification_setting.level] - end + notification = member_notification_setting(member) + ::NotificationSetting.levels[notification.level] if notification + end + + private + + def member_notification_setting(member) + member.user.notification_settings.select do |notification| + notification.source_id == member.source_id && notification.source_type == member.source_type + end.last end end @@ -680,18 +705,21 @@ module API expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| if options.key?(:project_members) - (options[:project_members] || []).find { |member| member.source_id == project.id } - else - project.project_members.find_by(user_id: options[:current_user].id) + (options[:project_members] || []).select { |member| member.source_id == project.id }.last + elsif project.project_members.any? + # This is not the bet option to search in a CollectionProxy, but if + # we use find_by we will perform another query, even if the association + # is loaded + project.project_members.select { |project_member| project_member.user_id == options[:current_user].id }.last end end expose :group_access, using: Entities::GroupAccess do |project, options| if project.group if options.key?(:group_members) - (options[:group_members] || []).find { |member| member.source_id == project.namespace_id } - else - project.group.group_members.find_by(user_id: options[:current_user].id) + (options[:group_members] || []).select { |member| member.source_id == project.namespace_id }.last + elsif project.group.group_members.any? + project.group.group_members.select { |group_member| group_member.user_id == options[:current_user].id }.last end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 83c7898150a..b81f07a1770 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -55,19 +55,8 @@ module API def find_group_projects(params) group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute - projects = projects.preload(:project_feature, :route, :group) - .preload(namespace: [:route, :owner], - tags: :taggings, - project_group_links: :group, - fork_network: :root_project, - forked_project_link: :forked_from_project, - forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) projects = reorder_projects(projects) - projects = paginate(projects) - projects_with_forks = projects + projects.map(&:forked_from_project).compact - ::Projects::BatchForksCountService.new(projects_with_forks).refresh_cache - ::Projects::BatchOpenIssuesCountService.new(projects).refresh_cache - projects + paginate(projects) end def present_groups(params, groups) @@ -190,7 +179,8 @@ module API get ":id/projects" do projects = find_group_projects(params) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project - present projects, with: entity, current_user: current_user + + present entity.prepare_relation(projects), with: entity, current_user: current_user end desc 'Get a list of subgroups in this group.' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 4cd7e714aa2..12506203429 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -79,9 +79,9 @@ module API projects = projects.with_statistics if params[:statistics] projects = projects.with_issues_enabled if params[:with_issues_enabled] projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + projects = paginate(projects) if current_user - projects = projects.includes(:route, :taggings, namespace: :route) project_members = current_user.project_members group_members = current_user.group_members end @@ -95,7 +95,7 @@ module API ) options[:with] = Entities::BasicProjectDetails if params[:simple] - present paginate(projects), options + present options[:with].prepare_relation(projects), options end end diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb new file mode 100644 index 00000000000..57df6157225 --- /dev/null +++ b/lib/api/projects_relation_builder.rb @@ -0,0 +1,34 @@ +module API + module ProjectsRelationBuilder + extend ActiveSupport::Concern + + module ClassMethods + def prepare_relation(relation) + relation = preload_relation(relation) + execute_batch_counting(relation) + relation + end + + def preload_relation(relation) + raise NotImplementedError, 'self.preload_relation method must be defined and return a relation' + end + + def forks_counting_projects(projects_relation) + projects_relation + end + + def batch_forks_counting(projects_relation) + ::Projects::BatchForksCountService.new(forks_counting_projects(projects_relation)).refresh_cache + end + + def batch_open_issues_counting(projects_relation) + ::Projects::BatchOpenIssuesCountService.new(projects_relation).refresh_cache + end + + def execute_batch_counting(projects_relation) + batch_forks_counting(projects_relation) + batch_open_issues_counting(projects_relation) + end + end + end +end -- cgit v1.2.1 From 194f7bca9a286942bcd764c836180964e33a1e92 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 30 Nov 2017 12:52:38 +0100 Subject: Comments from code review applied. Also switched forked_from_project and ForkedProjectLinks to ForkNetworkMember --- lib/api/entities.rb | 75 ++++++++++++++++++++---------------- lib/api/projects.rb | 10 ++--- lib/api/projects_relation_builder.rb | 10 ++--- 3 files changed, 52 insertions(+), 43 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 8f41b930c0a..dada353a314 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -96,7 +96,7 @@ module API expose :star_count, :forks_count expose :created_at, :last_activity_at - def self.preload_relation(projects_relation) + def self.preload_relation(projects_relation, options = {}) projects_relation.preload(:project_feature, :route) .preload(namespace: [:route, :owner], tags: :taggings) @@ -134,18 +134,6 @@ module API expose :members do |project| expose_url(api_v4_projects_members_path(id: project.id)) end - - def self.preload_relation(projects_relation) - super(projects_relation).preload(:group) - .preload(project_group_links: :group, - fork_network: :root_project, - forked_project_link: :forked_from_project, - forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) - end - - def self.forks_counting_projects(projects_relation) - projects_relation + projects_relation.map(&:forked_from_project).compact - end end expose :archived?, as: :archived @@ -165,7 +153,9 @@ module API expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace, using: 'API::Entities::NamespaceBasic' - expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } + expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } do |project, options| + project.fork_network_member.forked_from_project + end expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -182,6 +172,20 @@ module API expose :printing_merge_request_link_enabled expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + + def self.preload_relation(projects_relation, options = {}) + relation = super(projects_relation).preload(:group) + .preload(project_group_links: :group, + fork_network: :root_project, + fork_network_member: [forked_from_project: [:route, namespace: :route, tags: :taggings]]) + + # Remove this preload once forked_project_links and forked_from_project models have been removed + relation.preload(forked_project_link: :forked_from_project) + end + + def self.forks_counting_projects(projects_relation) + projects_relation + projects_relation.map(&:fork_network_member).compact.map(&:forked_from_project).compact + end end class ProjectStatistics < Grape::Entity @@ -653,16 +657,10 @@ module API class MemberAccess < Grape::Entity expose :access_level expose :notification_level do |member, options| - notification = member_notification_setting(member) - ::NotificationSetting.levels[notification.level] if notification - end - - private - - def member_notification_setting(member) - member.user.notification_settings.select do |notification| - notification.source_id == member.source_id && notification.source_type == member.source_type - end.last + # binding.pry if member.id == 5 + if member.notification_setting + ::NotificationSetting.levels[member.notification_setting.level] + end end end @@ -705,25 +703,36 @@ module API expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| if options.key?(:project_members) - (options[:project_members] || []).select { |member| member.source_id == project.id }.last - elsif project.project_members.any? - # This is not the bet option to search in a CollectionProxy, but if - # we use find_by we will perform another query, even if the association - # is loaded - project.project_members.select { |project_member| project_member.user_id == options[:current_user].id }.last + (options[:project_members] || []).find { |member| member.source_id == project.id } + else + project.project_member(options[:current_user]) end end expose :group_access, using: Entities::GroupAccess do |project, options| if project.group if options.key?(:group_members) - (options[:group_members] || []).select { |member| member.source_id == project.namespace_id }.last - elsif project.group.group_members.any? - project.group.group_members.select { |group_member| group_member.user_id == options[:current_user].id }.last + (options[:group_members] || []).find { |member| member.source_id == project.namespace_id } + else + project.group.group_member(options[:current_user]) end end end end + + def self.preload_relation(projects_relation, options = {}) + relation = super(projects_relation, options) + + unless options.key?(:group_members) + relation = relation.preload(group: [group_members: [:source, user: [notification_settings: :source]]]) + end + + unless options.key?(:project_members) + relation = relation.preload(project_members: [:source, user: [notification_settings: :source]]) + end + + relation + end end class LabelBasic < Grape::Entity diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 12506203429..9e92ff1417e 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -82,20 +82,20 @@ module API projects = paginate(projects) if current_user - project_members = current_user.project_members - group_members = current_user.group_members + project_members = current_user.project_members.preload(:source, user: [notification_settings: :source]) + group_members = current_user.group_members.preload(:source, user: [notification_settings: :source]) end options = options.reverse_merge( with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, statistics: params[:statistics], - project_members: project_members, - group_members: group_members, + project_members: nil, + group_members: nil, current_user: current_user ) options[:with] = Entities::BasicProjectDetails if params[:simple] - present options[:with].prepare_relation(projects), options + present options[:with].prepare_relation(projects, options), options end end diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index 57df6157225..2e042d51c05 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -3,13 +3,13 @@ module API extend ActiveSupport::Concern module ClassMethods - def prepare_relation(relation) - relation = preload_relation(relation) - execute_batch_counting(relation) - relation + def prepare_relation(projects_relation, options = {}) + projects_relation = preload_relation(projects_relation, options) + execute_batch_counting(projects_relation) + projects_relation end - def preload_relation(relation) + def preload_relation(projects_relation, options = {}) raise NotImplementedError, 'self.preload_relation method must be defined and return a relation' end -- cgit v1.2.1 From 966f83f9653aa774ee82be65dbdae0c6e4f297da Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 30 Nov 2017 13:13:00 +0100 Subject: Undoing debugging change --- lib/api/projects.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 9e92ff1417e..14a4fc6f025 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -89,8 +89,8 @@ module API options = options.reverse_merge( with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, statistics: params[:statistics], - project_members: nil, - group_members: nil, + project_members: project_members, + group_members: group_members, current_user: current_user ) options[:with] = Entities::BasicProjectDetails if params[:simple] -- cgit v1.2.1 From fa6b0a36bd315a4777bb8b3229b89808b95a9489 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 30 Nov 2017 14:01:56 +0100 Subject: Changes after rebase --- lib/api/entities.rb | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index dada353a314..e0eb2ecfb73 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -87,14 +87,25 @@ module API expose :created_at end - class BasicProjectDetails < ProjectIdentity - expose :default_branch, :tag_list + class BasicProjectDetails < Grape::Entity + include ::API::ProjectsRelationBuilder + + expose :default_branch + + # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 + expose :tag_list do |project| + # project.tags.order(:name).pluck(:name) is the most suitable option + # to avoid loading all the ActiveRecord objects but, if we use it here + # it override the preloaded associations and makes a query + # (fixed in https://github.com/rails/rails/pull/25976). + project.tags.map(&:name).sort + end expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :avatar_url do |project, options| project.avatar_url(only_path: false) end expose :star_count, :forks_count - expose :created_at, :last_activity_at + expose :last_activity_at def self.preload_relation(projects_relation, options = {}) projects_relation.preload(:project_feature, :route) -- cgit v1.2.1 From c7e7f4444c791eb0ad409310394954578aa232c6 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 30 Nov 2017 16:20:13 +0100 Subject: Removing blank line --- lib/api/entities.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e0eb2ecfb73..910ac4f1814 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -91,7 +91,6 @@ module API include ::API::ProjectsRelationBuilder expose :default_branch - # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 expose :tag_list do |project| # project.tags.order(:name).pluck(:name) is the most suitable option -- cgit v1.2.1 From c0c0926accdc3c49fc2a75a0eec2f96a8a5ad15c Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 30 Nov 2017 18:24:31 +0100 Subject: Removed binding.pry --- lib/api/entities.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 910ac4f1814..248e234580f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -667,7 +667,6 @@ module API class MemberAccess < Grape::Entity expose :access_level expose :notification_level do |member, options| - # binding.pry if member.id == 5 if member.notification_setting ::NotificationSetting.levels[member.notification_setting.level] end -- cgit v1.2.1 From fe95de88551bd3c8d22591764d948205f9fbc10e Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 30 Nov 2017 19:09:25 +0100 Subject: Fixed BasicProjectDetail parent --- lib/api/entities.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 248e234580f..d224f468c18 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -87,7 +87,7 @@ module API expose :created_at end - class BasicProjectDetails < Grape::Entity + class BasicProjectDetails < ProjectIdentity include ::API::ProjectsRelationBuilder expose :default_branch -- cgit v1.2.1 From 3527d1ff2bc06ba38e820b300e49f817d2833379 Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Thu, 30 Nov 2017 22:17:17 +0100 Subject: Undoing the change to ForkNetworkMember --- lib/api/entities.rb | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index d224f468c18..7cec8da013d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -163,9 +163,7 @@ module API expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace, using: 'API::Entities::NamespaceBasic' - expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } do |project, options| - project.fork_network_member.forked_from_project - end + expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -184,17 +182,15 @@ module API expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics def self.preload_relation(projects_relation, options = {}) - relation = super(projects_relation).preload(:group) - .preload(project_group_links: :group, - fork_network: :root_project, - fork_network_member: [forked_from_project: [:route, namespace: :route, tags: :taggings]]) - - # Remove this preload once forked_project_links and forked_from_project models have been removed - relation.preload(forked_project_link: :forked_from_project) + super(projects_relation).preload(:group) + .preload(project_group_links: :group, + fork_network: :root_project, + forked_project_link: :forked_from_project, + forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) end def self.forks_counting_projects(projects_relation) - projects_relation + projects_relation.map(&:fork_network_member).compact.map(&:forked_from_project).compact + projects_relation + projects_relation.map(&:forked_from_project).compact end end -- cgit v1.2.1 From 979056e964827a9d6efc979843ac567a3dd5cdfd Mon Sep 17 00:00:00 2001 From: Francisco Lopez Date: Fri, 1 Dec 2017 19:21:04 +0100 Subject: Not forcing to redefine preload_relation --- lib/api/projects_relation_builder.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index 2e042d51c05..6482fd94ab8 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -10,7 +10,7 @@ module API end def preload_relation(projects_relation, options = {}) - raise NotImplementedError, 'self.preload_relation method must be defined and return a relation' + projects_relation end def forks_counting_projects(projects_relation) -- cgit v1.2.1 From 3f45662e5af2f1cb477d2e7e08965f9ffd0304f7 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Fri, 1 Dec 2017 14:01:15 -0600 Subject: Don't disable the Rails mailer when seeding the test environment --- lib/gitlab/seeder.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index f9ab9bd466f..30df7e4a831 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -8,7 +8,8 @@ end module Gitlab class Seeder def self.quiet - mute_mailer + mute_mailer unless Rails.env.test? + SeedFu.quiet = true yield -- cgit v1.2.1 From d6435b68c4fc4e0325ec6a3deb807d0e3dd4dec4 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 6 Nov 2017 13:44:30 -0800 Subject: Add TrackUntrackedUploads post-deploy migration To create the table, and schedule the background migration that begins the work. --- .../prepare_unhashed_uploads.rb | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 lib/gitlab/background_migration/prepare_unhashed_uploads.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb new file mode 100644 index 00000000000..3efc604fa86 --- /dev/null +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -0,0 +1,35 @@ +module Gitlab + module BackgroundMigration + class PrepareUnhashedUploads + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + end + + def perform + return unless migrate? + + clear_unhashed_upload_files + store_unhashed_upload_files + schedule_populate_untracked_uploads_jobs + end + + private + + def migrate? + UnhashedUploadFile.table_exists? + end + + def clear_unhashed_upload_files + # TODO + end + + def store_unhashed_upload_files + # TODO + end + + def schedule_populate_untracked_uploads_jobs + # TODO + end + end + end +end -- cgit v1.2.1 From ab814e4dd3cf06559a95ca5dd19722431314f6fa Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:53:10 -0800 Subject: Backport `which` from EE --- lib/gitlab/utils.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index abb3d3a02c3..b3baaf036d8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -46,5 +46,22 @@ module Gitlab def random_string Random.rand(Float::MAX.to_i).to_s(36) end + + # See: http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby + # Cross-platform way of finding an executable in the $PATH. + # + # which('ruby') #=> /usr/bin/ruby + def which(cmd, env = ENV) + exts = env['PATHEXT'] ? env['PATHEXT'].split(';') : [''] + + env['PATH'].split(File::PATH_SEPARATOR).each do |path| + exts.each do |ext| + exe = File.join(path, "#{cmd}#{ext}") + return exe if File.executable?(exe) && !File.directory?(exe) + end + end + + nil + end end end -- cgit v1.2.1 From b6ea41d13073ce8b4d16b2edb602c82aae10ea0b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 6 Nov 2017 17:07:35 -0800 Subject: Find and store unhashed upload file paths --- .../prepare_unhashed_uploads.rb | 58 +++++++++++++++++++--- 1 file changed, 52 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 3efc604fa86..11f43044829 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -1,6 +1,9 @@ module Gitlab module BackgroundMigration class PrepareUnhashedUploads + FILE_PATH_BATCH_SIZE = 500 + UPLOAD_DIR = "#{CarrierWave.root}/uploads" + class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' end @@ -8,8 +11,8 @@ module Gitlab def perform return unless migrate? - clear_unhashed_upload_files - store_unhashed_upload_files + clear_unhashed_upload_file_paths + store_unhashed_upload_file_paths schedule_populate_untracked_uploads_jobs end @@ -19,12 +22,55 @@ module Gitlab UnhashedUploadFile.table_exists? end - def clear_unhashed_upload_files - # TODO + def clear_unhashed_upload_file_paths + UnhashedUploadFile.delete_all end - def store_unhashed_upload_files - # TODO + def store_unhashed_upload_file_paths + return unless Dir.exists?(UPLOAD_DIR) + + file_paths = [] + each_file_path(UPLOAD_DIR) do |file_path| + file_paths << file_path + + if file_paths.size >= FILE_PATH_BATCH_SIZE + insert_file_paths(file_paths) + file_paths = [] + end + end + + insert_file_paths(file_paths) if file_paths.any? + end + + def each_file_path(search_dir, &block) + cmd = build_find_command(search_dir) + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + stdout.each_line("\0") do |line| + yield(line.chomp("\0")) + end + raise "Find command failed" unless status_thread.value.success? + end + end + + def build_find_command(search_dir) + cmd = ['find', search_dir, '-type', 'f', '!', '-path', "#{UPLOAD_DIR}/@hashed/*", '!', '-path', "#{UPLOAD_DIR}/tmp/*", '-print0'] + + ['ionice', '-c', 'Idle'] + cmd if ionice_is_available? + + cmd + end + + def ionice_is_available? + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, even though it isn't very nice + false + end + + def insert_file_paths(file_paths) + file_paths.each do |file_path| + UnhashedUploadFile.create!(path: file_path) + end end def schedule_populate_untracked_uploads_jobs -- cgit v1.2.1 From 8315c66a569bbc1b4806762e4da49c22813fc523 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 12:53:24 -0800 Subject: Kick off follow up background migration jobs To process the unhashed_upload_files table. --- .../populate_untracked_uploads.rb | 51 ++++++++++++++++++++++ .../prepare_unhashed_uploads.rb | 8 +++- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/background_migration/populate_untracked_uploads.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb new file mode 100644 index 00000000000..6dbef41cff8 --- /dev/null +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -0,0 +1,51 @@ +module Gitlab + module BackgroundMigration + class PopulateUntrackedUploads + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + + scope :untracked, -> { where(tracked: false) } + + def ensure_tracked! + # TODO + # unless unhashed_upload_file.in_uploads? + # unhashed_upload_file.add_to_uploads + # end + # + # unhashed_upload_file.mark_as_tracked + end + + def model_id + # TODO + end + + def model_type + # TODO + end + + def uploader + # TODO + end + end + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + end + + def perform(start_id, end_id) + return unless migrate? + + files = UnhashedUploadFile.untracked.where(id: start_id..end_id) + files.each do |unhashed_upload_file| + unhashed_upload_file.ensure_tracked! + end + end + + private + + def migrate? + UnhashedUploadFile.table_exists? && Upload.table_exists? + end + end + end +end diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 11f43044829..556457039fa 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -1,10 +1,16 @@ module Gitlab module BackgroundMigration class PrepareUnhashedUploads + # For bulk_queue_background_migration_jobs_by_range + include Database::MigrationHelpers + FILE_PATH_BATCH_SIZE = 500 UPLOAD_DIR = "#{CarrierWave.root}/uploads" + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' class UnhashedUploadFile < ActiveRecord::Base + include EachBatch + self.table_name = 'unhashed_upload_files' end @@ -74,7 +80,7 @@ module Gitlab end def schedule_populate_untracked_uploads_jobs - # TODO + bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) end end end -- cgit v1.2.1 From 3a0ad99d59506592e8d5c6abf0de0fc2104f0bf2 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 19:08:02 -0800 Subject: Add untracked files to uploads --- .../populate_untracked_uploads.rb | 144 +++++++++++++++++++-- 1 file changed, 133 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 6dbef41cff8..acd424f4558 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -4,27 +4,149 @@ module Gitlab class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' + # Ends with /:random_hex/:filename + FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/ + + # These regex patterns are tested against a relative path, relative to + # the upload directory. + # For convenience, if there exists a capture group in the pattern, then + # it indicates the model_id. + PATH_PATTERNS = [ + { + pattern: /\A-\/system\/appearance\/logo\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Appearance', + }, + { + pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Appearance', + }, + { + pattern: /\A-\/system\/note\/attachment\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Note', + }, + { + pattern: /\A-\/system\/user\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'User', + }, + { + pattern: /\A-\/system\/group\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'Group', + }, + { + pattern: /\A-\/system\/project\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'Project', + }, + { + pattern: FILE_UPLOADER_PATH_PATTERN, + uploader: 'FileUploader', + model_type: 'Project' + }, + ] + scope :untracked, -> { where(tracked: false) } def ensure_tracked! - # TODO - # unless unhashed_upload_file.in_uploads? - # unhashed_upload_file.add_to_uploads - # end - # - # unhashed_upload_file.mark_as_tracked + return if persisted? && tracked? + + unless in_uploads? + add_to_uploads + end + + mark_as_tracked end - def model_id - # TODO + def in_uploads? + # Even though we are checking relative paths, path is enough to + # uniquely identify uploads. There is no ambiguity between + # FileUploader paths and other Uploader paths because we use the /-/ + # separator kind of like an escape character. Project full_path will + # never conflict with an upload path starting with "uploads/-/". + Upload.exists?(path: upload_path) end - def model_type - # TODO + def add_to_uploads + Upload.create!( + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + ) + end + + def mark_as_tracked + self.tracked = true + self.save! + end + + def upload_path + # UnhashedUploadFile#path is absolute, but Upload#path depends on uploader + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) + matchd[0].sub(/\A\//, '') # remove leading slash + else + path_relative_to_carrierwave_root + end end def uploader - # TODO + PATH_PATTERNS.each do |path_pattern_map| + if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + return path_pattern_map[:uploader] + end + end + end + + def model_type + PATH_PATTERNS.each do |path_pattern_map| + if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + return path_pattern_map[:model_type] + end + end + end + + def model_id + PATH_PATTERNS.each do |path_pattern_map| + matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + + # If something is captured (matchd[1] is not nil), it is a model_id + return matchd[1] if matchd && matchd[1] + end + + # Only the FileUploader pattern will not match an ID + file_uploader_model_id + end + + def file_size + File.size(path) + end + + # Not including a leading slash + def path_relative_to_upload_dir + @path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '') + end + + # Not including a leading slash + def path_relative_to_carrierwave_root + "uploads/#{path_relative_to_upload_dir}" + end + + private + + def file_uploader_model_id + pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ + matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) + raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + full_path = matchd[1] + project = Project.find_by_full_path(full_path) + project.id.to_s end end -- cgit v1.2.1 From 1bae010b63f0bcff79f32ce99190f2d5b6d9fbcd Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 19:54:28 -0800 Subject: Calculate checksums by copy-pasting in the whole `Upload` class. Also, fix `Namespace` `model_type` (it should not be `Group`). --- .../populate_untracked_uploads.rb | 67 +++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index acd424f4558..1773b53bd68 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -35,7 +35,7 @@ module Gitlab { pattern: /\A-\/system\/group\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Group', + model_type: 'Namespace', }, { pattern: /\A-\/system\/project\/avatar\/(\d+)/, @@ -150,8 +150,71 @@ module Gitlab end end + # Copy-pasted class for less fragile migration class Upload < ActiveRecord::Base - self.table_name = 'uploads' + self.table_name = 'uploads' # This is the only line different from copy-paste + + # Upper limit for foreground checksum processing + CHECKSUM_THRESHOLD = 100.megabytes + + belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + + validates :size, presence: true + validates :path, presence: true + validates :model, presence: true + validates :uploader, presence: true + + before_save :calculate_checksum, if: :foreground_checksum? + after_commit :schedule_checksum, unless: :foreground_checksum? + + def self.remove_path(path) + where(path: path).destroy_all + end + + def self.record(uploader) + remove_path(uploader.relative_path) + + create( + size: uploader.file.size, + path: uploader.relative_path, + model: uploader.model, + uploader: uploader.class.to_s + ) + end + + def absolute_path + return path unless relative_path? + + uploader_class.absolute_path(self) + end + + def calculate_checksum + return unless exist? + + self.checksum = Digest::SHA256.file(absolute_path).hexdigest + end + + def exist? + File.exist?(absolute_path) + end + + private + + def foreground_checksum? + size <= CHECKSUM_THRESHOLD + end + + def schedule_checksum + UploadChecksumWorker.perform_async(id) + end + + def relative_path? + !path.start_with?('/') + end + + def uploader_class + Object.const_get(uploader) + end end def perform(start_id, end_id) -- cgit v1.2.1 From 3dc74378ec54afb4b15d9f1bdf3781ddd50f83e3 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 20:15:28 -0800 Subject: Allow individual failures --- .../populate_untracked_uploads.rb | 40 +++++++++++++--------- 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 1773b53bd68..934431ccddd 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -97,28 +97,18 @@ module Gitlab end def uploader - PATH_PATTERNS.each do |path_pattern_map| - if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) - return path_pattern_map[:uploader] - end - end + matching_pattern_map[:uploader] end def model_type - PATH_PATTERNS.each do |path_pattern_map| - if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) - return path_pattern_map[:model_type] - end - end + matching_pattern_map[:model_type] end def model_id - PATH_PATTERNS.each do |path_pattern_map| - matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) - # If something is captured (matchd[1] is not nil), it is a model_id - return matchd[1] if matchd && matchd[1] - end + # If something is captured (matchd[1] is not nil), it is a model_id + return matchd[1] if matchd[1] # Only the FileUploader pattern will not match an ID file_uploader_model_id @@ -140,6 +130,16 @@ module Gitlab private + def matching_pattern_map + @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map| + path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + end + + raise "Unknown upload path pattern \"#{path}\"" unless @matching_pattern_map + + @matching_pattern_map + end + def file_uploader_model_id pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) @@ -222,7 +222,15 @@ module Gitlab files = UnhashedUploadFile.untracked.where(id: start_id..end_id) files.each do |unhashed_upload_file| - unhashed_upload_file.ensure_tracked! + begin + unhashed_upload_file.ensure_tracked! + rescue StandardError => e + Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" + + # The untracked rows will remain in the DB. We will be able to see + # which ones failed to become tracked, and then we can decide what + # to do. + end end end -- cgit v1.2.1 From ffbaf19fe8a83f651f45380749ccc12cd38ec29f Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 20:54:54 -0800 Subject: Fix Rubocop offenses --- .../background_migration/populate_untracked_uploads.rb | 16 ++++++++-------- .../background_migration/prepare_unhashed_uploads.rb | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 934431ccddd..ef0f1209ef5 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -15,39 +15,39 @@ module Gitlab { pattern: /\A-\/system\/appearance\/logo\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Appearance', + model_type: 'Appearance' }, { pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Appearance', + model_type: 'Appearance' }, { pattern: /\A-\/system\/note\/attachment\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Note', + model_type: 'Note' }, { pattern: /\A-\/system\/user\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'User', + model_type: 'User' }, { pattern: /\A-\/system\/group\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Namespace', + model_type: 'Namespace' }, { pattern: /\A-\/system\/project\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Project', + model_type: 'Project' }, { pattern: FILE_UPLOADER_PATH_PATTERN, uploader: 'FileUploader', model_type: 'Project' - }, - ] + } + ].freeze scope :untracked, -> { where(tracked: false) } diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 556457039fa..7c426022304 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -5,8 +5,8 @@ module Gitlab include Database::MigrationHelpers FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads" - FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' + UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze class UnhashedUploadFile < ActiveRecord::Base include EachBatch @@ -33,7 +33,7 @@ module Gitlab end def store_unhashed_upload_file_paths - return unless Dir.exists?(UPLOAD_DIR) + return unless Dir.exist?(UPLOAD_DIR) file_paths = [] each_file_path(UPLOAD_DIR) do |file_path| -- cgit v1.2.1 From 41412fec5b9aea8a6104320c5b554ffdabe52506 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:31:51 -0800 Subject: Avoid instantiating an AR object and ignore dupes --- .../background_migration/prepare_unhashed_uploads.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 7c426022304..982c0ff5320 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -75,10 +75,24 @@ module Gitlab def insert_file_paths(file_paths) file_paths.each do |file_path| - UnhashedUploadFile.create!(path: file_path) + insert_file_path(file_path) end end + def insert_file_path(file_path) + table_columns_and_values = 'unhashed_upload_files (path, created_at, updated_at) VALUES (?, ?, ?)' + + sql = if Gitlab::Database.postgresql? + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end + + timestamp = Time.now.utc.iso8601 + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + ActiveRecord::Base.connection.execute(sql) + end + def schedule_populate_untracked_uploads_jobs bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) end -- cgit v1.2.1 From 7c43692f68dd62773b0a7b094453f582a4242ef7 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:44:49 -0800 Subject: Make regexes more readable --- .../populate_untracked_uploads.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ef0f1209ef5..42ad28400f4 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -5,7 +5,8 @@ module Gitlab self.table_name = 'unhashed_upload_files' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/ + FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} + FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN = %r{\A(.+)#{FILE_UPLOADER_PATH_PATTERN}} # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -13,32 +14,32 @@ module Gitlab # it indicates the model_id. PATH_PATTERNS = [ { - pattern: /\A-\/system\/appearance\/logo\/(\d+)/, + pattern: %r{\A-/system/appearance/logo/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Appearance' }, { - pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, + pattern: %r{\A-/system/appearance/header_logo/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Appearance' }, { - pattern: /\A-\/system\/note\/attachment\/(\d+)/, + pattern: %r{\A-/system/note/attachment/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Note' }, { - pattern: /\A-\/system\/user\/avatar\/(\d+)/, + pattern: %r{\A-/system/user/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'User' }, { - pattern: /\A-\/system\/group\/avatar\/(\d+)/, + pattern: %r{\A-/system/group/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'Namespace' }, { - pattern: /\A-\/system\/project\/avatar\/(\d+)/, + pattern: %r{\A-/system/project/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'Project' }, @@ -90,7 +91,7 @@ module Gitlab if uploader == 'FileUploader' # Path relative to project directory in uploads matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(/\A\//, '') # remove leading slash + matchd[0].sub(%r{\A/}, '') # remove leading slash else path_relative_to_carrierwave_root end @@ -120,7 +121,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - @path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '') + @path_relative_to_upload_dir ||= path.sub(%r{\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/}, '') end # Not including a leading slash @@ -141,8 +142,7 @@ module Gitlab end def file_uploader_model_id - pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ - matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd full_path = matchd[1] project = Project.find_by_full_path(full_path) -- cgit v1.2.1 From 0e9efa74a7ea653a969436cb98c0a4ba80e8dd71 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 13:29:03 -0800 Subject: Use `find` `-prune` option for performance --- lib/gitlab/background_migration/prepare_unhashed_uploads.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 982c0ff5320..ce488542df9 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -59,9 +59,11 @@ module Gitlab end def build_find_command(search_dir) - cmd = ['find', search_dir, '-type', 'f', '!', '-path', "#{UPLOAD_DIR}/@hashed/*", '!', '-path', "#{UPLOAD_DIR}/tmp/*", '-print0'] + hashed_path = "#{UPLOAD_DIR}/@hashed/*" + tmp_path = "#{UPLOAD_DIR}/tmp/*" + cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - ['ionice', '-c', 'Idle'] + cmd if ionice_is_available? + %w[ionice -c Idle] + cmd if ionice_is_available? cmd end -- cgit v1.2.1 From 2ab3031bd35213802e508fef6eebceaaf40cee9b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 15:05:08 -0800 Subject: Refactor, no change in behavior --- .../populate_untracked_uploads.rb | 10 +++---- .../prepare_unhashed_uploads.rb | 35 +++++++++++++--------- 2 files changed, 25 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 42ad28400f4..e63220c8001 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -55,9 +55,7 @@ module Gitlab def ensure_tracked! return if persisted? && tracked? - unless in_uploads? - add_to_uploads - end + add_to_uploads unless in_uploads? mark_as_tracked end @@ -82,8 +80,7 @@ module Gitlab end def mark_as_tracked - self.tracked = true - self.save! + update!(tracked: true) end def upload_path @@ -121,7 +118,8 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - @path_relative_to_upload_dir ||= path.sub(%r{\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/}, '') + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR)}/} + @path_relative_to_upload_dir ||= path.sub(base, '') end # Not including a leading slash diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index ce488542df9..8033f994959 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -35,29 +35,36 @@ module Gitlab def store_unhashed_upload_file_paths return unless Dir.exist?(UPLOAD_DIR) - file_paths = [] - each_file_path(UPLOAD_DIR) do |file_path| - file_paths << file_path - - if file_paths.size >= FILE_PATH_BATCH_SIZE - insert_file_paths(file_paths) - file_paths = [] - end + each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + insert_file_paths(file_paths) end - - insert_file_paths(file_paths) if file_paths.any? end - def each_file_path(search_dir, &block) + def each_file_batch(search_dir, batch_size, &block) cmd = build_find_command(search_dir) + Open3.popen2(*cmd) do |stdin, stdout, status_thread| - stdout.each_line("\0") do |line| - yield(line.chomp("\0")) - end + yield_paths_in_batches(stdout, batch_size, &block) + raise "Find command failed" unless status_thread.value.success? end end + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) + end + def build_find_command(search_dir) hashed_path = "#{UPLOAD_DIR}/@hashed/*" tmp_path = "#{UPLOAD_DIR}/tmp/*" -- cgit v1.2.1 From a210cb6b827d9d918788578fc4ae956471de3b12 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:17:56 -0800 Subject: Rename table to untracked_files_for_uploads --- .../populate_untracked_uploads.rb | 16 +-- .../prepare_unhashed_uploads.rb | 110 --------------------- .../prepare_untracked_uploads.rb | 110 +++++++++++++++++++++ 3 files changed, 118 insertions(+), 118 deletions(-) delete mode 100644 lib/gitlab/background_migration/prepare_unhashed_uploads.rb create mode 100644 lib/gitlab/background_migration/prepare_untracked_uploads.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index e63220c8001..bd0f2f591a4 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -1,8 +1,8 @@ module Gitlab module BackgroundMigration class PopulateUntrackedUploads - class UnhashedUploadFile < ActiveRecord::Base - self.table_name = 'unhashed_upload_files' + class UntrackedFile < ActiveRecord::Base + self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} @@ -84,7 +84,7 @@ module Gitlab end def upload_path - # UnhashedUploadFile#path is absolute, but Upload#path depends on uploader + # UntrackedFile#path is absolute, but Upload#path depends on uploader if uploader == 'FileUploader' # Path relative to project directory in uploads matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) @@ -118,7 +118,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR)}/} + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end @@ -218,10 +218,10 @@ module Gitlab def perform(start_id, end_id) return unless migrate? - files = UnhashedUploadFile.untracked.where(id: start_id..end_id) - files.each do |unhashed_upload_file| + files = UntrackedFile.untracked.where(id: start_id..end_id) + files.each do |untracked_file| begin - unhashed_upload_file.ensure_tracked! + untracked_file.ensure_tracked! rescue StandardError => e Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" @@ -235,7 +235,7 @@ module Gitlab private def migrate? - UnhashedUploadFile.table_exists? && Upload.table_exists? + UntrackedFile.table_exists? && Upload.table_exists? end end end diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb deleted file mode 100644 index 8033f994959..00000000000 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ /dev/null @@ -1,110 +0,0 @@ -module Gitlab - module BackgroundMigration - class PrepareUnhashedUploads - # For bulk_queue_background_migration_jobs_by_range - include Database::MigrationHelpers - - FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze - FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - - class UnhashedUploadFile < ActiveRecord::Base - include EachBatch - - self.table_name = 'unhashed_upload_files' - end - - def perform - return unless migrate? - - clear_unhashed_upload_file_paths - store_unhashed_upload_file_paths - schedule_populate_untracked_uploads_jobs - end - - private - - def migrate? - UnhashedUploadFile.table_exists? - end - - def clear_unhashed_upload_file_paths - UnhashedUploadFile.delete_all - end - - def store_unhashed_upload_file_paths - return unless Dir.exist?(UPLOAD_DIR) - - each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| - insert_file_paths(file_paths) - end - end - - def each_file_batch(search_dir, batch_size, &block) - cmd = build_find_command(search_dir) - - Open3.popen2(*cmd) do |stdin, stdout, status_thread| - yield_paths_in_batches(stdout, batch_size, &block) - - raise "Find command failed" unless status_thread.value.success? - end - end - - def yield_paths_in_batches(stdout, batch_size, &block) - paths = [] - - stdout.each_line("\0") do |line| - paths << line.chomp("\0") - - if paths.size >= batch_size - yield(paths) - paths = [] - end - end - - yield(paths) - end - - def build_find_command(search_dir) - hashed_path = "#{UPLOAD_DIR}/@hashed/*" - tmp_path = "#{UPLOAD_DIR}/tmp/*" - cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - - %w[ionice -c Idle] + cmd if ionice_is_available? - - cmd - end - - def ionice_is_available? - Gitlab::Utils.which('ionice') - rescue StandardError - # In this case, returning false is relatively safe, even though it isn't very nice - false - end - - def insert_file_paths(file_paths) - file_paths.each do |file_path| - insert_file_path(file_path) - end - end - - def insert_file_path(file_path) - table_columns_and_values = 'unhashed_upload_files (path, created_at, updated_at) VALUES (?, ?, ?)' - - sql = if Gitlab::Database.postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end - - timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) - ActiveRecord::Base.connection.execute(sql) - end - - def schedule_populate_untracked_uploads_jobs - bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) - end - end - end -end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb new file mode 100644 index 00000000000..6a7fef18e53 --- /dev/null +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -0,0 +1,110 @@ +module Gitlab + module BackgroundMigration + class PrepareUntrackedUploads + # For bulk_queue_background_migration_jobs_by_range + include Database::MigrationHelpers + + FILE_PATH_BATCH_SIZE = 500 + UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + + class UntrackedFile < ActiveRecord::Base + include EachBatch + + self.table_name = 'untracked_files_for_uploads' + end + + def perform + return unless migrate? + + clear_untracked_file_paths + store_untracked_file_paths + schedule_populate_untracked_uploads_jobs + end + + private + + def migrate? + UntrackedFile.table_exists? + end + + def clear_untracked_file_paths + UntrackedFile.delete_all + end + + def store_untracked_file_paths + return unless Dir.exist?(UPLOAD_DIR) + + each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + insert_file_paths(file_paths) + end + end + + def each_file_batch(search_dir, batch_size, &block) + cmd = build_find_command(search_dir) + + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + yield_paths_in_batches(stdout, batch_size, &block) + + raise "Find command failed" unless status_thread.value.success? + end + end + + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) + end + + def build_find_command(search_dir) + hashed_path = "#{UPLOAD_DIR}/@hashed/*" + tmp_path = "#{UPLOAD_DIR}/tmp/*" + cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] + + %w[ionice -c Idle] + cmd if ionice_is_available? + + cmd + end + + def ionice_is_available? + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, even though it isn't very nice + false + end + + def insert_file_paths(file_paths) + file_paths.each do |file_path| + insert_file_path(file_path) + end + end + + def insert_file_path(file_path) + table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' + + sql = if Gitlab::Database.postgresql? + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end + + timestamp = Time.now.utc.iso8601 + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + ActiveRecord::Base.connection.execute(sql) + end + + def schedule_populate_untracked_uploads_jobs + bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) + end + end + end +end -- cgit v1.2.1 From c77a353dca25c2e08c409f1f3fc5a61a3eea69dc Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:21:32 -0800 Subject: Remove unnecessary clearing Since duplicate inserts are now ignored. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 ----- 1 file changed, 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 6a7fef18e53..11978b73a8a 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -17,7 +17,6 @@ module Gitlab def perform return unless migrate? - clear_untracked_file_paths store_untracked_file_paths schedule_populate_untracked_uploads_jobs end @@ -28,10 +27,6 @@ module Gitlab UntrackedFile.table_exists? end - def clear_untracked_file_paths - UntrackedFile.delete_all - end - def store_untracked_file_paths return unless Dir.exist?(UPLOAD_DIR) -- cgit v1.2.1 From 81f061d5a4ebefefe0efbc3c1f28b86b665a2b92 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 14:47:32 -0800 Subject: Fix `ionice` prepend --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 11978b73a8a..6aba3011720 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -65,7 +65,7 @@ module Gitlab tmp_path = "#{UPLOAD_DIR}/tmp/*" cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - %w[ionice -c Idle] + cmd if ionice_is_available? + cmd = %w[ionice -c Idle] + cmd if ionice_is_available? cmd end -- cgit v1.2.1 From 3dc0b118eca3716c0cda841232e0789739627957 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 16:11:53 -0800 Subject: Store paths relative to CarrierWave.root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So the path on source installs cannot be too long for our column. And fix the column length test since Route.path is limited to 255 chars, it doesn’t matter how many nested groups there are. --- .../background_migration/populate_untracked_uploads.rb | 12 ++++-------- .../background_migration/prepare_untracked_uploads.rb | 16 +++++++++------- 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index bd0f2f591a4..c5f9d998d9e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -90,7 +90,7 @@ module Gitlab matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) matchd[0].sub(%r{\A/}, '') # remove leading slash else - path_relative_to_carrierwave_root + path end end @@ -113,20 +113,16 @@ module Gitlab end def file_size - File.size(path) + absolute_path = File.join(CarrierWave.root, path) + File.size(absolute_path) end # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR)}/} + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end - # Not including a leading slash - def path_relative_to_carrierwave_root - "uploads/#{path_relative_to_upload_dir}" - end - private def matching_pattern_map diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 6aba3011720..e9d162661d1 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -5,8 +5,12 @@ module Gitlab include Database::MigrationHelpers FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + RELATIVE_UPLOAD_DIR = "uploads".freeze + ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/} + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze class UntrackedFile < ActiveRecord::Base include EachBatch @@ -28,9 +32,9 @@ module Gitlab end def store_untracked_file_paths - return unless Dir.exist?(UPLOAD_DIR) + return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) - each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + each_file_batch(ABSOLUTE_UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| insert_file_paths(file_paths) end end @@ -49,7 +53,7 @@ module Gitlab paths = [] stdout.each_line("\0") do |line| - paths << line.chomp("\0") + paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '') if paths.size >= batch_size yield(paths) @@ -61,9 +65,7 @@ module Gitlab end def build_find_command(search_dir) - hashed_path = "#{UPLOAD_DIR}/@hashed/*" - tmp_path = "#{UPLOAD_DIR}/tmp/*" - cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] + cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] cmd = %w[ionice -c Idle] + cmd if ionice_is_available? -- cgit v1.2.1 From 0715034805a8f68c66118cf78777bca92ad7cef1 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:20:15 -0800 Subject: Remove irrelevant copy-pasted code --- .../populate_untracked_uploads.rb | 20 -------------------- 1 file changed, 20 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index c5f9d998d9e..f2207fa4e72 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -153,29 +153,9 @@ module Gitlab belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations - validates :size, presence: true - validates :path, presence: true - validates :model, presence: true - validates :uploader, presence: true - before_save :calculate_checksum, if: :foreground_checksum? after_commit :schedule_checksum, unless: :foreground_checksum? - def self.remove_path(path) - where(path: path).destroy_all - end - - def self.record(uploader) - remove_path(uploader.relative_path) - - create( - size: uploader.file.size, - path: uploader.relative_path, - model: uploader.model, - uploader: uploader.class.to_s - ) - end - def absolute_path return path unless relative_path? -- cgit v1.2.1 From b63e8f4adfda2f907280824e6acf69bbaa56de3a Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:49:10 -0800 Subject: Fallback on checksum jobs Since `calculate_checksum` depends on `Uploader` classes which are not defined in this background migration and may change at any time. --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f2207fa4e72..8529e8d1d0b 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -166,6 +166,8 @@ module Gitlab return unless exist? self.checksum = Digest::SHA256.file(absolute_path).hexdigest + rescue StandardError + schedule_checksum end def exist? -- cgit v1.2.1 From c25b7c0e3f6d43b5fb77e53bbd0dd4495b8e0c69 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:49:24 -0800 Subject: Speed up inserts --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index e9d162661d1..983e63143e0 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -80,8 +80,10 @@ module Gitlab end def insert_file_paths(file_paths) - file_paths.each do |file_path| - insert_file_path(file_path) + ActiveRecord::Base.transaction do + file_paths.each do |file_path| + insert_file_path(file_path) + end end end -- cgit v1.2.1 From dd8680a7ae4be279ae1d90f0889317a1e6ee0d95 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 02:36:25 -0800 Subject: Drop temporary tracking table when finished --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 8529e8d1d0b..f28892174bb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -208,6 +208,8 @@ module Gitlab # to do. end end + + drop_temp_table_if_finished end private @@ -215,6 +217,10 @@ module Gitlab def migrate? UntrackedFile.table_exists? && Upload.table_exists? end + + def drop_temp_table_if_finished + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.untracked.empty? + end end end end -- cgit v1.2.1 From 7fd26434196df01091b18747f91f54c0701bb292 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 03:01:35 -0800 Subject: Fix Rubocop offenses --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 983e63143e0..9c40cf8aee2 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -91,13 +91,13 @@ module Gitlab table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' sql = if Gitlab::Database.postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend ActiveRecord::Base.connection.execute(sql) end -- cgit v1.2.1 From 10c660be007406533e48d5e3c6485ecf210e051b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 04:51:28 -0800 Subject: Fix migration for pre-Postgres 9.5 --- .../background_migration/prepare_untracked_uploads.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 9c40cf8aee2..8772092da64 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -88,9 +88,14 @@ module Gitlab end def insert_file_path(file_path) + if postgresql_pre_9_5? + # No easy way to do ON CONFLICT DO NOTHING before Postgres 9.5 so just use Rails + return UntrackedFile.where(path: file_path).first_or_create + end + table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' - sql = if Gitlab::Database.postgresql? + sql = if postgresql? "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" else "INSERT IGNORE INTO #{table_columns_and_values};" @@ -101,6 +106,15 @@ module Gitlab ActiveRecord::Base.connection.execute(sql) end + def postgresql? + @postgresql ||= Gitlab::Database.postgresql? + end + + def postgresql_pre_9_5? + @postgresql_pre_9_5 ||= postgresql? && + ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 + end + def schedule_populate_untracked_uploads_jobs bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) end -- cgit v1.2.1 From 87529ce5823036d4b9dd9ca412643befc8e490c3 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 05:19:07 -0800 Subject: Move temp table creation into the prepare job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Hopefully fixes spec failures in which the table doesn’t exist * Decouples the background migration from the post-deploy migration, e.g. we could easily run it again even though the table is dropped when finished. --- .../prepare_untracked_uploads.rb | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 8772092da64..e0c1daaccf7 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -19,16 +19,29 @@ module Gitlab end def perform - return unless migrate? - + ensure_temporary_tracking_table_exists store_untracked_file_paths schedule_populate_untracked_uploads_jobs end private - def migrate? - UntrackedFile.table_exists? + def ensure_temporary_tracking_table_exists + unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) + UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| + t.string :path, limit: 600, null: false + t.boolean :tracked, default: false, null: false + t.timestamps_with_timezone null: false + end + end + + unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :path) + UntrackedFile.connection.add_index :untracked_files_for_uploads, :path, unique: true + end + + unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :tracked) + UntrackedFile.connection.add_index :untracked_files_for_uploads, :tracked + end end def store_untracked_file_paths -- cgit v1.2.1 From 5552d1adf4489b71190c6b59ab66b300fd5a0604 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 16 Nov 2017 16:24:42 -0800 Subject: Log the find command used --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index e0c1daaccf7..853c9810359 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -82,6 +82,8 @@ module Gitlab cmd = %w[ionice -c Idle] + cmd if ionice_is_available? + Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + cmd end -- cgit v1.2.1 From 17ce21d74eab4d2973d372cb3f97258eb3b81de9 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 17 Nov 2017 13:49:25 -0800 Subject: Use ionice absolute path --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 853c9810359..c3f5dddb07d 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -80,14 +80,15 @@ module Gitlab def build_find_command(search_dir) cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] - cmd = %w[ionice -c Idle] + cmd if ionice_is_available? + ionice = which_ionice + cmd = %W[#{ionice} -c Idle] + cmd if ionice Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" cmd end - def ionice_is_available? + def which_ionice Gitlab::Utils.which('ionice') rescue StandardError # In this case, returning false is relatively safe, even though it isn't very nice -- cgit v1.2.1 From edb5cac46c1cba1029fb3e67d4853027590584f6 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 20 Nov 2017 16:27:24 -0800 Subject: Use bulk inserts --- .../prepare_untracked_uploads.rb | 55 ++++++++++++++-------- 1 file changed, 35 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index c3f5dddb07d..022b2f41393 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -20,7 +20,19 @@ module Gitlab def perform ensure_temporary_tracking_table_exists + + # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since + # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be + # slow, start with an empty table for Postgres < 9.5. + # That way we can do bulk inserts at ~30x the speed of individual + # inserts (~20 minutes worth of inserts at GitLab.com scale instead of + # ~10 hours). + # In all other cases, installations will get both bulk inserts and the + # ability for these jobs to retry without having to clear and reinsert. + clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates? + store_untracked_file_paths + schedule_populate_untracked_uploads_jobs end @@ -44,6 +56,10 @@ module Gitlab end end + def clear_untracked_file_paths + UntrackedFile.delete_all + end + def store_untracked_file_paths return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) @@ -96,36 +112,35 @@ module Gitlab end def insert_file_paths(file_paths) - ActiveRecord::Base.transaction do - file_paths.each do |file_path| - insert_file_path(file_path) - end - end - end + sql = if postgresql_pre_9_5? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" + elsif postgresql? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)} ON CONFLICT DO NOTHING;" + else # MySQL + "INSERT IGNORE INTO #{table_columns_and_values_for_insert(file_paths)};" + end - def insert_file_path(file_path) - if postgresql_pre_9_5? - # No easy way to do ON CONFLICT DO NOTHING before Postgres 9.5 so just use Rails - return UntrackedFile.where(path: file_path).first_or_create - end + ActiveRecord::Base.connection.execute(sql) + end - table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' + def table_columns_and_values_for_insert(file_paths) + timestamp = Time.now.utc.iso8601 - sql = if postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end + values = file_paths.map do |file_path| + ActiveRecord::Base.send(:sanitize_sql_array, ['(?, ?, ?)', file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend + end.join(', ') - timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend - ActiveRecord::Base.connection.execute(sql) + "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" end def postgresql? @postgresql ||= Gitlab::Database.postgresql? end + def can_bulk_insert_and_ignore_duplicates? + !postgresql_pre_9_5? + end + def postgresql_pre_9_5? @postgresql_pre_9_5 ||= postgresql? && ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 -- cgit v1.2.1 From aefbdcdcf25cbcd8d80dcaa3216264d91bdcac91 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 21 Nov 2017 16:05:33 -0800 Subject: Fix Rubocop offense --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f28892174bb..b8872477e63 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -138,6 +138,7 @@ module Gitlab def file_uploader_model_id matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + full_path = matchd[1] project = Project.find_by_full_path(full_path) project.id.to_s -- cgit v1.2.1 From 8def25d9f163cc22f71716d8a22fc5adfe0a762e Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 21 Nov 2017 16:05:57 -0800 Subject: Fix datetime inserts on MySQL --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 022b2f41393..f5a11658c0b 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -124,10 +124,8 @@ module Gitlab end def table_columns_and_values_for_insert(file_paths) - timestamp = Time.now.utc.iso8601 - values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?, ?, ?)', file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?, NOW(), NOW())', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" -- cgit v1.2.1 From a9155a94fe29aa67230f2e5ef3d6393345677ce0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 10:23:24 -0800 Subject: Refactor --- .../background_migration/prepare_untracked_uploads.rb | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index f5a11658c0b..8333a6218de 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -44,16 +44,10 @@ module Gitlab t.string :path, limit: 600, null: false t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false + t.index :path, unique: true + t.index :tracked end end - - unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :path) - UntrackedFile.connection.add_index :untracked_files_for_uploads, :path, unique: true - end - - unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :tracked) - UntrackedFile.connection.add_index :untracked_files_for_uploads, :tracked - end end def clear_untracked_file_paths @@ -140,8 +134,7 @@ module Gitlab end def postgresql_pre_9_5? - @postgresql_pre_9_5 ||= postgresql? && - ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 + @postgresql_pre_9_5 ||= postgresql? && Gitlab::Database.version.to_f < 9.5 end def schedule_populate_untracked_uploads_jobs -- cgit v1.2.1 From 67b58ffdc357bb94ec62d696b7b9a4aecf751c75 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 10:44:33 -0800 Subject: Get rid of tracked field It makes a debugging slightly easier, but is not necessary, and is a waste of resources. --- .../background_migration/populate_untracked_uploads.rb | 14 +++----------- .../background_migration/prepare_untracked_uploads.rb | 2 -- 2 files changed, 3 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index b8872477e63..03e7b7b71cb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -50,14 +50,10 @@ module Gitlab } ].freeze - scope :untracked, -> { where(tracked: false) } - def ensure_tracked! - return if persisted? && tracked? - add_to_uploads unless in_uploads? - mark_as_tracked + delete end def in_uploads? @@ -79,10 +75,6 @@ module Gitlab ) end - def mark_as_tracked - update!(tracked: true) - end - def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader if uploader == 'FileUploader' @@ -197,7 +189,7 @@ module Gitlab def perform(start_id, end_id) return unless migrate? - files = UntrackedFile.untracked.where(id: start_id..end_id) + files = UntrackedFile.where(id: start_id..end_id) files.each do |untracked_file| begin untracked_file.ensure_tracked! @@ -220,7 +212,7 @@ module Gitlab end def drop_temp_table_if_finished - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.untracked.empty? + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? end end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 8333a6218de..c076c13815d 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -42,10 +42,8 @@ module Gitlab unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| t.string :path, limit: 600, null: false - t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false t.index :path, unique: true - t.index :tracked end end end -- cgit v1.2.1 From 7549d17f721b3be84f83c1dfa491d6a2ebf4ec28 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 12:19:25 -0800 Subject: Refactor --- .../populate_untracked_uploads.rb | 23 ++++++++++------------ 1 file changed, 10 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 03e7b7b71cb..41dc5a3ed7e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -51,28 +51,25 @@ module Gitlab ].freeze def ensure_tracked! - add_to_uploads unless in_uploads? + add_to_uploads_if_needed delete end - def in_uploads? + def add_to_uploads_if_needed # Even though we are checking relative paths, path is enough to # uniquely identify uploads. There is no ambiguity between # FileUploader paths and other Uploader paths because we use the /-/ # separator kind of like an escape character. Project full_path will # never conflict with an upload path starting with "uploads/-/". - Upload.exists?(path: upload_path) - end - - def add_to_uploads - Upload.create!( - path: upload_path, - uploader: uploader, - model_type: model_type, - model_id: model_id, - size: file_size - ) + Upload. + where(path: upload_path). + first_or_create!( + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + ) end def upload_path -- cgit v1.2.1 From 908aacddda571bc82c722798f399fd5a68ebe1da Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:12:24 -0800 Subject: Filter existing uploads with one query --- .../populate_untracked_uploads.rb | 80 ++++++++++++---------- 1 file changed, 42 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 41dc5a3ed7e..8d6da0a7a67 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -50,37 +50,25 @@ module Gitlab } ].freeze - def ensure_tracked! - add_to_uploads_if_needed - - delete - end - - def add_to_uploads_if_needed - # Even though we are checking relative paths, path is enough to - # uniquely identify uploads. There is no ambiguity between - # FileUploader paths and other Uploader paths because we use the /-/ - # separator kind of like an escape character. Project full_path will - # never conflict with an upload path starting with "uploads/-/". - Upload. - where(path: upload_path). - first_or_create!( - uploader: uploader, - model_type: model_type, - model_id: model_id, - size: file_size - ) + def to_h + { + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + } end def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader - if uploader == 'FileUploader' - # Path relative to project directory in uploads - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(%r{\A/}, '') # remove leading slash - else - path - end + @upload_path ||= if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) + matchd[0].sub(%r{\A/}, '') # remove leading slash + else + path + end end def uploader @@ -187,17 +175,8 @@ module Gitlab return unless migrate? files = UntrackedFile.where(id: start_id..end_id) - files.each do |untracked_file| - begin - untracked_file.ensure_tracked! - rescue StandardError => e - Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" - - # The untracked rows will remain in the DB. We will be able to see - # which ones failed to become tracked, and then we can decide what - # to do. - end - end + insert_uploads_if_needed(files) + files.delete_all drop_temp_table_if_finished end @@ -208,6 +187,31 @@ module Gitlab UntrackedFile.table_exists? && Upload.table_exists? end + def insert_uploads_if_needed(files) + filtered_files = filter_existing_uploads(files) + filtered_files = filter_deleted_models(filtered_files) + insert(filtered_files) + end + + def filter_existing_uploads(files) + paths = files.map(&:upload_path) + existing_paths = Upload.where(path: paths).pluck(:path).to_set + + files.reject do |file| + existing_paths.include?(file.upload_path) + end + end + + def filter_deleted_models(files) + files # TODO + end + + def insert(files) + files.each do |file| + Upload.create!(file.to_h) + end + end + def drop_temp_table_if_finished UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? end -- cgit v1.2.1 From a9c868d111c0231c4358d0ee017d1a9e9a28d3dd Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:49:16 -0800 Subject: Bulk insert uploads --- .../populate_untracked_uploads.rb | 63 +++++----------------- 1 file changed, 14 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 8d6da0a7a67..9fb5b3f77a2 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -56,7 +56,8 @@ module Gitlab uploader: uploader, model_type: model_type, model_id: model_id, - size: file_size + size: file_size, + checksum: checksum } end @@ -90,10 +91,13 @@ module Gitlab end def file_size - absolute_path = File.join(CarrierWave.root, path) File.size(absolute_path) end + def checksum + Digest::SHA256.file(absolute_path).hexdigest + end + # Not including a leading slash def path_relative_to_upload_dir base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} @@ -120,55 +124,14 @@ module Gitlab project = Project.find_by_full_path(full_path) project.id.to_s end - end - - # Copy-pasted class for less fragile migration - class Upload < ActiveRecord::Base - self.table_name = 'uploads' # This is the only line different from copy-paste - - # Upper limit for foreground checksum processing - CHECKSUM_THRESHOLD = 100.megabytes - - belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations - - before_save :calculate_checksum, if: :foreground_checksum? - after_commit :schedule_checksum, unless: :foreground_checksum? def absolute_path - return path unless relative_path? - - uploader_class.absolute_path(self) - end - - def calculate_checksum - return unless exist? - - self.checksum = Digest::SHA256.file(absolute_path).hexdigest - rescue StandardError - schedule_checksum - end - - def exist? - File.exist?(absolute_path) - end - - private - - def foreground_checksum? - size <= CHECKSUM_THRESHOLD - end - - def schedule_checksum - UploadChecksumWorker.perform_async(id) - end - - def relative_path? - !path.start_with?('/') + File.join(CarrierWave.root, path) end + end - def uploader_class - Object.const_get(uploader) - end + class Upload < ActiveRecord::Base + self.table_name = 'uploads' end def perform(start_id, end_id) @@ -207,9 +170,11 @@ module Gitlab end def insert(files) - files.each do |file| - Upload.create!(file.to_h) + rows = files.map do |file| + file.to_h.merge(created_at: 'NOW()') end + + Gitlab::Database.bulk_insert('uploads', rows) end def drop_temp_table_if_finished -- cgit v1.2.1 From 61a73cadb7f21de9f863fc1a16f13880861ac9f4 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:51:29 -0800 Subject: Get rid of timestamps on untracked files table `updated_at` is now unnecessary and `created_at` is less useful due to removing the tracked field. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index c076c13815d..358b76d39fb 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -42,7 +42,6 @@ module Gitlab unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| t.string :path, limit: 600, null: false - t.timestamps_with_timezone null: false t.index :path, unique: true end end @@ -117,10 +116,10 @@ module Gitlab def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?, NOW(), NOW())', file_path]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') - "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" + "#{UntrackedFile.table_name} (path) VALUES #{values}" end def postgresql? -- cgit v1.2.1 From 473ddfb453d820f1a32fb48477e17ba45bdbd2f0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 00:49:04 -0800 Subject: =?UTF-8?q?Don=E2=80=99t=20recreate=20deleted=20uploads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../populate_untracked_uploads.rb | 40 +++++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 9fb5b3f77a2..50ec2260c60 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -81,13 +81,13 @@ module Gitlab end def model_id + return @model_id if defined?(@model_id) + matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) # If something is captured (matchd[1] is not nil), it is a model_id - return matchd[1] if matchd[1] - # Only the FileUploader pattern will not match an ID - file_uploader_model_id + @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id end def file_size @@ -122,7 +122,9 @@ module Gitlab full_path = matchd[1] project = Project.find_by_full_path(full_path) - project.id.to_s + return nil unless project + + project.id end def absolute_path @@ -165,8 +167,36 @@ module Gitlab end end + # There are files on disk that are not in the uploads table because their + # model was deleted, and we don't delete the files on disk. def filter_deleted_models(files) - files # TODO + ids = deleted_model_ids(files) + + files.reject do |file| + ids[file.model_type].include?(file.model_id) + end + end + + def deleted_model_ids(files) + ids = { + 'Appearance' => [], + 'Namespace' => [], + 'Note' => [], + 'Project' => [], + 'User' => [] + } + + # group model IDs by model type + files.each do |file| + ids[file.model_type] << file.model_id + end + + ids.each do |model_type, model_ids| + found_ids = Object.const_get(model_type).where(id: model_ids.uniq).pluck(:id) + ids[model_type] = ids[model_type] - found_ids # replace with deleted ids + end + + ids end def insert(files) -- cgit v1.2.1 From 6e36452e96139658ff8eae88209710438dd14eba Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 00:52:16 -0800 Subject: Refactor --- .../background_migration/populate_untracked_uploads.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 50ec2260c60..f06b4ac1940 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -98,12 +98,6 @@ module Gitlab Digest::SHA256.file(absolute_path).hexdigest end - # Not including a leading slash - def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} - @path_relative_to_upload_dir ||= path.sub(base, '') - end - private def matching_pattern_map @@ -127,6 +121,12 @@ module Gitlab project.id end + # Not including a leading slash + def path_relative_to_upload_dir + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} + @path_relative_to_upload_dir ||= path.sub(base, '') + end + def absolute_path File.join(CarrierWave.root, path) end -- cgit v1.2.1 From 3694fe0f3d43619174878805ba09f0943e1dbaad Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 10:04:44 -0800 Subject: =?UTF-8?q?Don=E2=80=99t=20quote=20`NOW()`=20for=20created=5Fat=20?= =?UTF-8?q?column?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To fix for MySQL. --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 2 +- lib/gitlab/database.rb | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f06b4ac1940..ea04484c6a1 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -204,7 +204,7 @@ module Gitlab file.to_h.merge(created_at: 'NOW()') end - Gitlab::Database.bulk_insert('uploads', rows) + Gitlab::Database.bulk_insert('uploads', rows, disable_quote: :created_at) end def drop_temp_table_if_finished diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index cd7b4c043da..16308b308b2 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -116,15 +116,21 @@ module Gitlab # values. # return_ids - When set to true the return value will be an Array of IDs of # the inserted rows, this only works on PostgreSQL. - def self.bulk_insert(table, rows, return_ids: false) + # disable_quote - A key or an Array of keys to exclude from quoting (You + # become responsible for protection from SQL injection for + # these keys!) + def self.bulk_insert(table, rows, return_ids: false, disable_quote: []) return if rows.empty? keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } return_ids = false if mysql? + disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| - row.values_at(*keys).map { |value| connection.quote(value) } + row.keys.map do |k| + disable_quote.include?(k) ? row[k] : connection.quote(row[k]) + end end sql = <<-EOF -- cgit v1.2.1 From e5cf23dfcf22f5a82ec027212d0a178894ea2396 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 20:37:23 -0800 Subject: Ensure consistent column order --- lib/gitlab/database.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 16308b308b2..4c40e8a528c 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -128,7 +128,7 @@ module Gitlab disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| - row.keys.map do |k| + keys.map do |k| disable_quote.include?(k) ? row[k] : connection.quote(row[k]) end end -- cgit v1.2.1 From 74b3870a958cadf35fd3c13a78334c96d46de939 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 22:57:21 -0800 Subject: Address Rubocop offenses --- .../populate_untracked_uploads.rb | 64 +++++++++++++++------- .../prepare_untracked_uploads.rb | 57 +++++++++++++------ 2 files changed, 82 insertions(+), 39 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ea04484c6a1..802b661886b 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -1,12 +1,18 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration - class PopulateUntrackedUploads - class UntrackedFile < ActiveRecord::Base + # This class processes a batch of rows in `untracked_files_for_uploads` by + # adding each file to the `uploads` table if it does not exist. + class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength + # This class is responsible for producing the attributes necessary to + # track an uploaded file in the `uploads` table. + class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} - FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN = %r{\A(.+)#{FILE_UPLOADER_PATH_PATTERN}} + FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} + FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}} # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -44,7 +50,7 @@ module Gitlab model_type: 'Project' }, { - pattern: FILE_UPLOADER_PATH_PATTERN, + pattern: FILE_UPLOADER_PATH, uploader: 'FileUploader', model_type: 'Project' } @@ -63,13 +69,14 @@ module Gitlab def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader - @upload_path ||= if uploader == 'FileUploader' - # Path relative to project directory in uploads - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(%r{\A/}, '') # remove leading slash - else - path - end + @upload_path ||= + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH) + matchd[0].sub(%r{\A/}, '') # remove leading slash + else + path + end end def uploader @@ -83,7 +90,8 @@ module Gitlab def model_id return @model_id if defined?(@model_id) - matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) + pattern = matching_pattern_map[:pattern] + matchd = path_relative_to_upload_dir.match(pattern) # If something is captured (matchd[1] is not nil), it is a model_id # Only the FileUploader pattern will not match an ID @@ -105,14 +113,20 @@ module Gitlab path_relative_to_upload_dir.match(path_pattern_map[:pattern]) end - raise "Unknown upload path pattern \"#{path}\"" unless @matching_pattern_map + unless @matching_pattern_map + raise "Unknown upload path pattern \"#{path}\"" + end @matching_pattern_map end def file_uploader_model_id - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) - raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE) + not_found_msg = <<~MSG + Could not capture project full_path from a FileUploader path: + "#{path_relative_to_upload_dir}" + MSG + raise not_found_msg unless matchd full_path = matchd[1] project = Project.find_by_full_path(full_path) @@ -123,7 +137,8 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} + upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength + base = %r{\A#{Regexp.escape(upload_dir)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end @@ -132,6 +147,7 @@ module Gitlab end end + # This class is used to query the `uploads` table. class Upload < ActiveRecord::Base self.table_name = 'uploads' end @@ -192,8 +208,10 @@ module Gitlab end ids.each do |model_type, model_ids| - found_ids = Object.const_get(model_type).where(id: model_ids.uniq).pluck(:id) - ids[model_type] = ids[model_type] - found_ids # replace with deleted ids + model_class = Object.const_get(model_type) + found_ids = model_class.where(id: model_ids.uniq).pluck(:id) + deleted_ids = ids[model_type] - found_ids + ids[model_type] = deleted_ids end ids @@ -204,11 +222,15 @@ module Gitlab file.to_h.merge(created_at: 'NOW()') end - Gitlab::Database.bulk_insert('uploads', rows, disable_quote: :created_at) + Gitlab::Database.bulk_insert('uploads', + rows, + disable_quote: :created_at) end def drop_temp_table_if_finished - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? + if UntrackedFile.all.empty? + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) + end end end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 358b76d39fb..d8cddd98cbb 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -1,10 +1,14 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration - class PrepareUntrackedUploads + # This class finds all non-hashed uploaded file paths and saves them to a + # `untracked_files_for_uploads` table. + class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength # For bulk_queue_background_migration_jobs_by_range include Database::MigrationHelpers - FILE_PATH_BATCH_SIZE = 500 + FIND_BATCH_SIZE = 500 RELATIVE_UPLOAD_DIR = "uploads".freeze ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze @@ -12,6 +16,8 @@ module Gitlab EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + # This class is used to iterate over batches of + # `untracked_files_for_uploads` rows. class UntrackedFile < ActiveRecord::Base include EachBatch @@ -39,8 +45,9 @@ module Gitlab private def ensure_temporary_tracking_table_exists - unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) - UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| + table_name = :untracked_files_for_uploads + unless UntrackedFile.connection.table_exists?(table_name) + UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true end @@ -54,7 +61,7 @@ module Gitlab def store_untracked_file_paths return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) - each_file_batch(ABSOLUTE_UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths| insert_file_paths(file_paths) end end @@ -85,12 +92,17 @@ module Gitlab end def build_find_command(search_dir) - cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] + cmd = %W[find #{search_dir} + -type f + ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) + -print0] ionice = which_ionice cmd = %W[#{ionice} -c Idle] + cmd if ionice - Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + Rails.logger.info log_msg cmd end @@ -98,25 +110,32 @@ module Gitlab def which_ionice Gitlab::Utils.which('ionice') rescue StandardError - # In this case, returning false is relatively safe, even though it isn't very nice + # In this case, returning false is relatively safe, + # even though it isn't very nice false end def insert_file_paths(file_paths) - sql = if postgresql_pre_9_5? - "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - elsif postgresql? - "INSERT INTO #{table_columns_and_values_for_insert(file_paths)} ON CONFLICT DO NOTHING;" - else # MySQL - "INSERT IGNORE INTO #{table_columns_and_values_for_insert(file_paths)};" - end + sql = insert_sql(file_paths) ActiveRecord::Base.connection.execute(sql) end + def insert_sql(file_paths) + if postgresql_pre_9_5? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" + elsif postgresql? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ + " ON CONFLICT DO NOTHING;" + else # MySQL + "INSERT IGNORE INTO"\ + " #{table_columns_and_values_for_insert(file_paths)};" + end + end + def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength end.join(', ') "#{UntrackedFile.table_name} (path) VALUES #{values}" @@ -131,11 +150,13 @@ module Gitlab end def postgresql_pre_9_5? - @postgresql_pre_9_5 ||= postgresql? && Gitlab::Database.version.to_f < 9.5 + @postgresql_pre_9_5 ||= postgresql? && + Gitlab::Database.version.to_f < 9.5 end def schedule_populate_untracked_uploads_jobs - bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) + bulk_queue_background_migration_jobs_by_range( + UntrackedFile, FOLLOW_UP_MIGRATION) end end end -- cgit v1.2.1 From ca817816801c26847b66d105d07fd4473b015b31 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 27 Nov 2017 09:33:13 -0800 Subject: Handle race condition --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 802b661886b..ebb483c3cff 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -229,7 +229,8 @@ module Gitlab def drop_temp_table_if_finished if UntrackedFile.all.empty? - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) + UntrackedFile.connection.drop_table(:untracked_files_for_uploads, + if_exists: true) end end end -- cgit v1.2.1 From 25df666156279e5b392b429519b4f4ba01eefaac Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 19 Sep 2017 09:14:06 +0200 Subject: Create Ci::Artifacts To allow jobs/builds to have multiple artifacts, and to start seperating concerns from Ci::Build a new model is created: Ci::Artifact. Changes include the updating of the ArtifactUploader to adapt to a slightly different interface. The uploader expects to be initialized with a `Ci::Build`. Futher a migration with the minimal fields, the needed foreign keys and an index. Last, the way this works is by prepending a module to Ci::Build so we can basically override behaviour but if needed use `super` to get the original behaviour. --- lib/api/entities.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index ce332fe85d2..9eb2c98c436 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1036,7 +1036,7 @@ module API expose :steps, using: Step expose :image, using: Image expose :services, using: Service - expose :artifacts, using: Artifacts + expose :artifacts_options, as: :artifacts, using: Artifacts expose :cache, using: Cache expose :credentials, using: Credentials expose :dependencies, using: Dependency -- cgit v1.2.1 From 61864a5a5bb523953589c9398a431c4369fbfc76 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 21 Sep 2017 10:34:12 +0200 Subject: Rename Artifact to JobArtifact, split metadata out Two things at ones, as there was no clean way to seperate the commit and give me feedback from the tests. But the model Artifact is now JobArtifact, and the table does not have a type anymore, but the metadata is now its own model: Ci::JobArtifactMetadata. --- lib/api/entities.rb | 8 ++------ lib/api/runner.rb | 7 ++++--- 2 files changed, 6 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9eb2c98c436..0964bd98fbb 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1006,13 +1006,9 @@ module API expose :type, :url, :username, :password end - class ArtifactFile < Grape::Entity - expose :filename, :size - end - class Dependency < Grape::Entity expose :id, :name, :token - expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? } + expose :artifacts_file, using: JobArtifactFile, if: ->(job, _) { job.artifacts? } end class Response < Grape::Entity @@ -1036,7 +1032,7 @@ module API expose :steps, using: Step expose :image, using: Image expose :services, using: Service - expose :artifacts_options, as: :artifacts, using: Artifacts + expose :artifacts, using: Artifacts expose :cache, using: Cache expose :credentials, using: Credentials expose :dependencies, using: Dependency diff --git a/lib/api/runner.rb b/lib/api/runner.rb index a3987c560dd..8de0868c063 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -215,15 +215,16 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts_upload_path = JobArtifactUploader.artifacts_upload_path artifacts = uploaded_file(:file, artifacts_upload_path) metadata = uploaded_file(:metadata, artifacts_upload_path) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size - job.artifacts_file = artifacts - job.artifacts_metadata = metadata + job.job_artifacts.build(project: job.project, file_type: :archive, file: artifacts) + job.job_artifacts.build(project: job.project, file_type: :metadata, file: metadata) + job.artifacts_expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in -- cgit v1.2.1 From 871de0f18581bb03fed5c0d800f8183598a0195f Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 23 Nov 2017 16:57:27 +0100 Subject: Rename artifacts_* to legacy_artifacts_* --- lib/backup/artifacts.rb | 2 +- lib/gitlab/workhorse.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 1f4bda6f588..7a582a20056 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', ArtifactUploader.local_artifacts_store) + super('artifacts', LegacyArtifactUploader.local_store_path) end def create_files_dir diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 864a9e04888..c3e2742306d 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -58,7 +58,7 @@ module Gitlab end def artifact_upload_ok - { TempPath: ArtifactUploader.artifacts_upload_path } + { TempPath: LegacyArtifactUploader.artifacts_upload_path } end def send_git_blob(repository, blob) -- cgit v1.2.1 From f5d3b92155b76d2459014372e00d877b21408f49 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Mon, 27 Nov 2017 14:31:26 +0100 Subject: Remove Ci::Build#artifacts_file? --- lib/gitlab/workhorse.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index c3e2742306d..5ab6cd5a4ef 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -58,7 +58,7 @@ module Gitlab end def artifact_upload_ok - { TempPath: LegacyArtifactUploader.artifacts_upload_path } + { TempPath: JobArtifactUploader.artifacts_upload_path } end def send_git_blob(repository, blob) -- cgit v1.2.1 From 0464c25f602dc2e4d147ca2ac714d49bf96ddcf2 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Sun, 3 Dec 2017 12:02:11 +0100 Subject: Store expire_at in ci_job_artifacts --- lib/api/runner.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 8de0868c063..80feb629d54 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -222,12 +222,13 @@ module API bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size - job.job_artifacts.build(project: job.project, file_type: :archive, file: artifacts) - job.job_artifacts.build(project: job.project, file_type: :metadata, file: metadata) - - job.artifacts_expire_in = params['expire_in'] || + expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in + job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, expire_in: expire_in) + job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata + job.artifacts_expire_in = expire_in + if job.save present job, with: Entities::JobRequest::Response else -- cgit v1.2.1 From 50c8bd6350ebfd8d35deb2a1b41eb36e193d1a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 4 Dec 2017 00:53:04 +0100 Subject: Generate user agent header for GCP Client --- lib/google_api/cloud_platform/client.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 9242cbe840c..615cd7dc60a 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -82,6 +82,10 @@ module GoogleApi def token_life_time(expires_at) DateTime.strptime(expires_at, '%s').to_time.utc - Time.now.utc end + + def user_agent_header + { 'User-Agent': "GitLab/#{Gitlab::VERSION.match('(\d+\.\d+)').captures.first} (GPN:GitLab;)" } + end end end end -- cgit v1.2.1 From 04c6d102616b48c95b09656efc720c7dfdc99d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 4 Dec 2017 01:59:29 +0100 Subject: Use RequestOptions in GCP Client user_agent_header --- lib/google_api/cloud_platform/client.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 615cd7dc60a..15401057903 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -84,7 +84,9 @@ module GoogleApi end def user_agent_header - { 'User-Agent': "GitLab/#{Gitlab::VERSION.match('(\d+\.\d+)').captures.first} (GPN:GitLab;)" } + options = Google::Apis::RequestOptions.new + options.header = { 'User-Agent': "GitLab/#{Gitlab::VERSION.match('(\d+\.\d+)').captures.first} (GPN:GitLab;)" } + options end end end -- cgit v1.2.1 From 68b43f4d9c5dbc2d8264e5cadb7417e21ac0ed27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 4 Dec 2017 02:19:02 +0100 Subject: Test usage of custom user agent in GCP Client --- lib/google_api/cloud_platform/client.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 15401057903..36be6b7e97a 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -44,7 +44,7 @@ module GoogleApi service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token - service.get_zone_cluster(project_id, zone, cluster_id) + service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header) end def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:) @@ -62,14 +62,14 @@ module GoogleApi } } ) - service.create_cluster(project_id, zone, request_body) + service.create_cluster(project_id, zone, request_body, options: user_agent_header) end def projects_zones_operations(project_id, zone, operation_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token - service.get_zone_operation(project_id, zone, operation_id) + service.get_zone_operation(project_id, zone, operation_id, options: user_agent_header) end def parse_operation_id(self_link) -- cgit v1.2.1 From 327a9898a226a098b18e80e4950702064ecd38f1 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 30 Nov 2017 21:34:31 +0000 Subject: Fix the fork project functionality for projects with hashed storage Note the dependency on gitlab-shell v5.10.0 --- lib/gitlab/shell.rb | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 996d213fdb4..a22a63665be 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -143,20 +143,27 @@ module Gitlab storage, "#{path}.git", "#{new_path}.git"]) end - # Fork repository to new namespace + # Fork repository to new path # forked_from_storage - forked-from project's storage path - # path - project path with namespace + # forked_from_disk_path - project disk path # forked_to_storage - forked-to project's storage path - # fork_namespace - namespace for forked project + # forked_to_disk_path - forked project disk path # # Ex. - # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") + # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") # # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. - def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) - gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', - forked_from_storage, "#{path}.git", forked_to_storage, - fork_namespace]) + def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) + gitlab_shell_fast_execute( + [ + gitlab_shell_projects_path, + 'fork-repository', + forked_from_storage, + "#{forked_from_disk_path}.git", + forked_to_storage, + "#{forked_to_disk_path}.git" + ] + ) end # Remove repository from file system -- cgit v1.2.1 From 7c2b7296d4e623ba4eaa19cf03405ee7f2ae1ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Mon, 4 Dec 2017 09:49:53 +0000 Subject: Added default order to UserFinder --- lib/api/helpers/pagination.rb | 10 ++++++++++ lib/api/users.rb | 2 ++ 2 files changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 95108292aac..bb70370ba77 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -2,6 +2,8 @@ module API module Helpers module Pagination def paginate(relation) + relation = add_default_order(relation) + relation.page(params[:page]).per(params[:per_page]).tap do |data| add_pagination_headers(data) end @@ -45,6 +47,14 @@ module API # Ensure there is in total at least 1 page [paginated_data.total_pages, 1].max end + + def add_default_order(relation) + if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? + relation = relation.order(:id) + end + + relation + end end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 0cd89b1bcf8..e5de31ad51b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -76,6 +76,8 @@ module API forbidden!("Not authorized to access /api/v4/users") unless authorized entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic + users = users.preload(:identities, :u2f_registrations) if entity == Entities::UserWithAdmin + present paginate(users), with: entity end -- cgit v1.2.1 From bb80d439f4bbb37032a2ee2d35ee5eab7594de7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 4 Dec 2017 13:43:45 +0100 Subject: Refactor GCP Client#user_agent_header to use #tap --- lib/google_api/cloud_platform/client.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 36be6b7e97a..b0563fb2d69 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -84,9 +84,9 @@ module GoogleApi end def user_agent_header - options = Google::Apis::RequestOptions.new - options.header = { 'User-Agent': "GitLab/#{Gitlab::VERSION.match('(\d+\.\d+)').captures.first} (GPN:GitLab;)" } - options + Google::Apis::RequestOptions.new.tap do |options| + options.header = { 'User-Agent': "GitLab/#{Gitlab::VERSION.match('(\d+\.\d+)').captures.first} (GPN:GitLab;)" } + end end end end -- cgit v1.2.1 From 9737f582a10ff43cac0ee47f20f200ffd744b93a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Dec 2017 13:56:32 +0100 Subject: Reduce pipeline chain life span to minimize side effects --- lib/gitlab/ci/pipeline/chain/sequence.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 015f2988327..e24630656d3 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -5,20 +5,19 @@ module Gitlab class Sequence def initialize(pipeline, command, sequence) @pipeline = pipeline + @command = command + @sequence = sequence @completed = [] - - @sequence = sequence.map do |chain| - chain.new(pipeline, command) - end end def build! - @sequence.each do |step| - step.perform! + @sequence.each do |chain| + step = chain.new(@pipeline, @command) + step.perform! break if step.break? - @completed << step + @completed.push(step) end @pipeline.tap do -- cgit v1.2.1 From 12f61e0d2cabb07172ef93143b084104141dd771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Mon, 4 Dec 2017 10:31:13 +0100 Subject: Move SingleRepositoryWorker#fsck into Gitlab::Git::Repository --- lib/gitlab/git/repository.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index dbc08747228..867fc2a42f6 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1118,9 +1118,11 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args, env: {}) + def run_git(args, env: {}, nice: false) + cmd = [Gitlab.config.git.bin_path, *args] + cmd.unshift("nice") if nice circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path, env) + popen(cmd, path, env) end end @@ -1187,6 +1189,12 @@ module Gitlab end end + def fsck + output, status = run_git(%W[--git-dir=#{path} fsck], nice: true) + + raise GitError.new("Could not fsck repository:\n#{output}") unless status.zero? + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end -- cgit v1.2.1 From 8be9567c64d2c90a56e36308f39ee89938137614 Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Tue, 21 Nov 2017 14:47:52 +0100 Subject: Migrate Gitlab::Git::Repository#cherry_pick to Gitaly Closes gitaly#737 --- lib/gitlab/git/commit.rb | 22 +++++++ lib/gitlab/git/repository.rb | 87 ++++++++++++++++----------- lib/gitlab/gitaly_client/operation_service.rb | 30 +++++++++ 3 files changed, 105 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index d5518814483..c85dcfa0475 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -418,6 +418,20 @@ module Gitlab parent_ids.size > 1 end + def to_gitaly_commit + return raw_commit if raw_commit.is_a?(Gitaly::GitCommit) + + message_split = raw_commit.message.split("\n", 2) + Gitaly::GitCommit.new( + id: raw_commit.oid, + subject: message_split[0] ? message_split[0].chomp.b : "", + body: raw_commit.message.b, + parent_ids: raw_commit.parent_ids, + author: gitaly_commit_author_from_rugged(raw_commit.author), + committer: gitaly_commit_author_from_rugged(raw_commit.committer) + ) + end + private def init_from_hash(hash) @@ -463,6 +477,14 @@ module Gitlab def serialize_keys SERIALIZE_KEYS end + + def gitaly_commit_author_from_rugged(author_or_committer) + Gitaly::CommitAuthor.new( + name: author_or_committer[:name].b, + email: author_or_committer[:email].b, + date: Google::Protobuf::Timestamp.new(seconds: author_or_committer[:time].to_i) + ) + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index dbc08747228..05b93a2be84 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -809,44 +809,24 @@ module Gitlab end def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) - raise CreateTreeError unless cherry_pick_tree_id - - committer = user_to_committer(user) + gitaly_migrate(:cherry_pick) do |is_enabled| + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - create_commit(message: message, - author: { - email: commit.author_email, - name: commit.author_name, - time: commit.authored_date - }, - committer: committer, - tree: cherry_pick_tree_id, - parents: [start_commit.sha]) + if is_enabled + gitaly_operations_client.user_cherry_pick(args) + else + rugged_cherry_pick(args) + end end end - def check_cherry_pick_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << 1 if target_commit.merge_commit? - - cherry_pick_index = rugged.cherrypick_commit(*args) - return false if cherry_pick_index.conflicts? - - tree_id = cherry_pick_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - def diff_exists?(sha1, sha2) rugged.diff(sha1, sha2).size > 0 end @@ -1665,6 +1645,45 @@ module Gitlab raise InvalidRef, ex end + def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) + raise CreateTreeError unless cherry_pick_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: { + email: commit.author_email, + name: commit.author_name, + time: commit.authored_date + }, + committer: committer, + tree: cherry_pick_tree_id, + parents: [start_commit.sha]) + end + end + + def check_cherry_pick_content(target_commit, source_sha) + args = [target_commit.sha, source_sha] + args << 1 if target_commit.merge_commit? + + cherry_pick_index = rugged.cherrypick_commit(*args) + return false if cherry_pick_index.conflicts? + + tree_id = cherry_pick_index.write_tree(rugged) + return false unless diff_exists?(source_sha, tree_id) + + tree_id + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 526d44a8b77..51de6a9a75d 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -122,6 +122,36 @@ module Gitlab ).branch_update Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) end + + def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + request = Gitaly::UserCherryPickRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + commit: commit.to_gitaly_commit, + branch_name: GitalyClient.encode(branch_name), + message: GitalyClient.encode(message), + start_branch_name: GitalyClient.encode(start_branch_name.to_s), + start_repository: start_repository.gitaly_repository + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_cherry_pick, + request, + remote_storage: start_repository.storage + ) + + if response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + elsif response.commit_error.presence + raise Gitlab::Git::CommitError, response.commit_error + elsif response.create_tree_error.presence + raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error + else + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + end end end end -- cgit v1.2.1 From f91c5c5bbe15adc1cc951bc734123c2994622f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Mon, 4 Dec 2017 12:58:24 -0300 Subject: Backport Squash/Rebase refactor from EE --- lib/gitlab/git/repository.rb | 148 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 867fc2a42f6..d50de2fb6c1 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -18,6 +18,8 @@ module Gitlab GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze SEARCH_CONTEXT_LINES = 3 + REBASE_WORKTREE_PREFIX = 'rebase'.freeze + SQUASH_WORKTREE_PREFIX = 'squash'.freeze NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -1090,13 +1092,8 @@ module Gitlab raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] input = "update #{ref_path}\x00#{ref}\x00\x00" - output, status = circuit_breaker.perform do - popen(command, path) { |stdin| stdin.write(input) } - end - - raise GitError, output unless status.zero? + run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } end def fetch_ref(source_repository, source_ref:, target_ref:) @@ -1118,14 +1115,22 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args, env: {}, nice: false) + def run_git(args, chdir: path, env: {}, nice: false, &block) cmd = [Gitlab.config.git.bin_path, *args] cmd.unshift("nice") if nice circuit_breaker.perform do - popen(cmd, path, env) + popen(cmd, chdir, env, &block) end end + def run_git!(args, chdir: path, env: {}, nice: false, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) + + raise GitError, output unless status.zero? + + output + end + # Refactoring aid; allows us to copy code from app/models/repository.rb def run_git_with_timeout(args, timeout, env: {}) circuit_breaker.perform do @@ -1195,6 +1200,64 @@ module Gitlab raise GitError.new("Could not fsck repository:\n#{output}") unless status.zero? end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repository.path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + + def rebase_in_progress?(rebase_id) + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + + def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) + squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) + env = git_env_for_user(user).merge( + 'GIT_AUTHOR_NAME' => author.name, + 'GIT_AUTHOR_EMAIL' => author.email + ) + diff_range = "#{start_sha}...#{end_sha}" + diff_files = run_git!( + %W(diff --name-only --diff-filter=a --binary #{diff_range}) + ).chomp + + with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do + # Apply diff of the `diff_range` to the worktree + diff = run_git!(%W(diff --binary #{diff_range})) + run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + stdin.write(diff) + end + + # Commit the `diff_range` diff + run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + + # Return the squash sha. May print a warning for ambiguous refs, but + # we can ignore that with `--quiet` and just take the SHA, if present. + # HEAD here always refers to the current HEAD commit, even if there is + # another ref called HEAD. + run_git!( + %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env + ).chomp + end + end + + def squash_in_progress?(squash_id) + fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1231,6 +1294,57 @@ module Gitlab private + def fresh_worktree?(path) + File.exist?(path) && !clean_stuck_worktree(path) + end + + def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) + base_args = %w(worktree add --detach) + + # Note that we _don't_ want to test for `.present?` here: If the caller + # passes an non nil empty value it means it still wants sparse checkout + # but just isn't interested in any file, perhaps because it wants to + # checkout files in by a changeset but that changeset only adds files. + if sparse_checkout_files + # Create worktree without checking out + run_git!(base_args + ['--no-checkout', worktree_path], env: env) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path) + + configure_sparse_checkout(worktree_git_path, sparse_checkout_files) + + # After sparse checkout configuration, checkout `branch` in worktree + run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) + else + # Create worktree and checkout `branch` in it + run_git!(base_args + [worktree_path, branch], env: env) + end + + yield + ensure + FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) + FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) + end + + def clean_stuck_worktree(path) + return false unless File.mtime(path) < 15.minutes.ago + + FileUtils.rm_rf(path) + true + end + + # Adding a worktree means checking out the repository. For large repos, + # this can be very expensive, so set up sparse checkout for the worktree + # to only check out the files we're interested in. + def configure_sparse_checkout(worktree_git_path, files) + run_git!(%w(config core.sparseCheckout true)) + + return if files.empty? + + worktree_info_path = File.join(worktree_git_path, 'info') + FileUtils.mkdir_p(worktree_info_path) + File.write(File.join(worktree_info_path, 'sparse-checkout'), files) + end + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) with_repo_branch_commit(source_repository, source_branch) do |commit| if commit @@ -1242,6 +1356,24 @@ module Gitlab end end + def worktree_path(prefix, id) + id = id.to_s + raise ArgumentError, "worktree id can't be empty" unless id.present? + raise ArgumentError, "worktree id can't contain slashes " if id.include?("/") + + File.join(path, 'gitlab-worktree', "#{prefix}-#{id}") + end + + def git_env_for_user(user) + { + 'GIT_COMMITTER_NAME' => user.name, + 'GIT_COMMITTER_EMAIL' => user.email, + 'GL_ID' => Gitlab::GlId.gl_id(user), + 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, + 'GL_REPOSITORY' => gl_repository + } + end + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 -- cgit v1.2.1 From 2286681e1ca5cfbdbb6167cc4f31f0933ee584a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Thu, 30 Nov 2017 20:04:57 -0300 Subject: Add missing attr_accessor to Gitlab::Git::Conflict::File --- lib/gitlab/git/conflict/file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index fc1595f1faf..b2a625e08fa 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,7 +2,7 @@ module Gitlab module Git module Conflict class File - attr_reader :content, :their_path, :our_path, :our_mode, :repository + attr_reader :content, :their_path, :our_path, :our_mode, :repository, :commit_oid def initialize(repository, commit_oid, conflict, content) @repository = repository -- cgit v1.2.1 From 359b65beac43e009b715c2db048e06b6f96b0ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Fri, 1 Dec 2017 18:15:52 -0300 Subject: Use `String#end_with?` instead of `String#ends_with?` The former is in Ruby's core lib, so is more flexible. --- lib/gitlab/git/conflict/resolver.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index df509c5f4ce..de8cce41b6d 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -75,7 +75,7 @@ module Gitlab resolved_lines = file.resolve_lines(params[:sections]) new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") - new_file << "\n" if file.our_blob.data.ends_with?("\n") + new_file << "\n" if file.our_blob.data.end_with?("\n") elsif params[:content] new_file = file.resolve_content(params[:content]) end -- cgit v1.2.1 From fcbd2fe62568c55c86de4eeb5969bfe4d82af6ce Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 4 Dec 2017 12:57:44 -0800 Subject: Follow symlinks In particular, the Omnibus uploads directory is generally a symlink. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index d8cddd98cbb..476c46341ae 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -92,7 +92,7 @@ module Gitlab end def build_find_command(search_dir) - cmd = %W[find #{search_dir} + cmd = %W[find -L #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -- cgit v1.2.1 From 8cce70730c2fb9c705e1f1177f6d1effc665b3c7 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Thu, 24 Aug 2017 08:20:36 +0200 Subject: Create merge request from email * new merge request can be created by sending an email to the specific email address (similar to creating issues by email) * for the first iteration, source branch must be specified in the mail subject, other merge request parameters can not be set yet * user should enable "Receive notifications about your own activity" in user settings to receive a notification about created merge request Part of #32878 --- lib/gitlab/email/handler.rb | 2 + .../email/handler/create_merge_request_handler.rb | 67 ++++++++++++++++++++++ lib/gitlab/email/receiver.rb | 6 +- 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 lib/gitlab/email/handler/create_merge_request_handler.rb (limited to 'lib') diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index b07c68d1498..e08b5be8984 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,3 +1,4 @@ +require 'gitlab/email/handler/create_merge_request_handler' require 'gitlab/email/handler/create_note_handler' require 'gitlab/email/handler/create_issue_handler' require 'gitlab/email/handler/unsubscribe_handler' @@ -8,6 +9,7 @@ module Gitlab HANDLERS = [ UnsubscribeHandler, CreateNoteHandler, + CreateMergeRequestHandler, CreateIssueHandler ].freeze diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb new file mode 100644 index 00000000000..c63666b98c1 --- /dev/null +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -0,0 +1,67 @@ +require 'gitlab/email/handler/base_handler' +require 'gitlab/email/handler/reply_processing' + +module Gitlab + module Email + module Handler + class CreateMergeRequestHandler < BaseHandler + include ReplyProcessing + attr_reader :project_path, :incoming_email_token + + def initialize(mail, mail_key) + super(mail, mail_key) + if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) + @project_path, @incoming_email_token = m.captures + end + end + + def can_handle? + @project_path && @incoming_email_token + end + + def execute + raise ProjectNotFound unless project + + validate_permission!(:create_merge_request) + + verify_record!( + record: create_merge_request, + invalid_exception: InvalidMergeRequestError, + record_name: 'merge_request') + end + + def author + @author ||= User.find_by(incoming_email_token: incoming_email_token) + end + + def project + @project ||= Project.find_by_full_path(project_path) + end + + def metrics_params + super.merge(project: project&.full_path) + end + + private + + def create_merge_request + merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute + + if merge_request.errors.any? + merge_request + else + MergeRequests::CreateService.new(project, author).create(merge_request) + end + end + + def merge_request_params + { + source_project_id: project.id, + source_branch: mail.subject, + target_project_id: project.id + } + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index c8f4591d060..d8c594ad0e7 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -13,8 +13,10 @@ module Gitlab UserBlockedError = Class.new(ProcessingError) UserNotAuthorizedError = Class.new(ProcessingError) NoteableNotFoundError = Class.new(ProcessingError) - InvalidNoteError = Class.new(ProcessingError) - InvalidIssueError = Class.new(ProcessingError) + InvalidRecordError = Class.new(ProcessingError) + InvalidNoteError = Class.new(InvalidRecordError) + InvalidIssueError = Class.new(InvalidRecordError) + InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) class Receiver -- cgit v1.2.1 From 0b15570e497d3c5c515be59a43b686087b985f5c Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 28 Nov 2017 17:08:30 +0100 Subject: Add ApplicationWorker and make every worker include it --- lib/gitlab/sidekiq_config.rb | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 lib/gitlab/sidekiq_config.rb (limited to 'lib') diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb new file mode 100644 index 00000000000..dc9886732b5 --- /dev/null +++ b/lib/gitlab/sidekiq_config.rb @@ -0,0 +1,50 @@ +require 'yaml' + +module Gitlab + module SidekiqConfig + def self.redis_queues + @redis_queues ||= Sidekiq::Queue.all.map(&:name) + end + + # This method is called by `bin/sidekiq-cluster` in EE, which runs outside + # of bundler/Rails context, so we cannot use any gem or Rails methods. + def self.config_queues(rails_path = Rails.root.to_s) + @config_queues ||= begin + config = YAML.load_file(File.join(rails_path, 'config', 'sidekiq_queues.yml')) + config[:queues].map(&:first) + end + end + + def self.cron_workers + @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize } + end + + def self.workers + @workers ||= find_workers(Rails.root.join('app', 'workers')) + end + + def self.default_queues + [ActionMailer::DeliveryJob.queue_name, 'default'] + end + + def self.worker_queues + @worker_queues ||= (workers.map(&:queue) + default_queues).uniq + end + + def self.find_workers(root) + concerns = root.join('concerns').to_s + + workers = Dir[root.join('**', '*.rb')] + .reject { |path| path.start_with?(concerns) } + + workers.map! do |path| + ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') + + ns.camelize.constantize + end + + # Skip concerns + workers.select { |w| w < Sidekiq::Worker } + end + end +end -- cgit v1.2.1 From 1e6ca3c41ead23c5e433460c8c807ea73d9ec0ef Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 29 Nov 2017 16:30:17 +0100 Subject: Consistently schedule Sidekiq jobs --- lib/after_commit_queue.rb | 26 ++++++++++++++++++++++++-- lib/gitlab/database/migration_helpers.rb | 4 ++-- 2 files changed, 26 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index 4750a2c373a..db63c5038ae 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -6,12 +6,34 @@ module AfterCommitQueue after_rollback :_clear_after_commit_queue end - def run_after_commit(method = nil, &block) - _after_commit_queue << proc { self.send(method) } if method # rubocop:disable GitlabSecurity/PublicSend + def run_after_commit(&block) _after_commit_queue << block if block + + true + end + + def run_after_commit_or_now(&block) + if AfterCommitQueue.inside_transaction? + run_after_commit(&block) + else + instance_eval(&block) + end + true end + def self.open_transactions_baseline + if ::Rails.env.test? + return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) } + end + + 0 + end + + def self.inside_transaction? + ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + end + protected def _run_after_commit_queue diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index c276c3566b4..3f65bc912de 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -703,14 +703,14 @@ into similar problems in the future (e.g. when new tables are created). # We push multiple jobs at a time to reduce the time spent in # Sidekiq/Redis operations. We're using this buffer based approach so we # don't need to run additional queries for every range. - BackgroundMigrationWorker.perform_bulk(jobs) + BackgroundMigrationWorker.bulk_perform_async(jobs) jobs.clear end jobs << [job_class_name, [start_id, end_id]] end - BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty? + BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty? end # Queues background migration jobs for an entire table, batched by ID range. -- cgit v1.2.1 From 887a3739163c32fd5b815dcbba4a59fc69b6db23 Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Mon, 4 Dec 2017 14:13:22 +0100 Subject: Migrate Gitlab::Git::Repository#revert to Gitaly Closes gitaly#780 --- lib/gitlab/git/repository.rb | 53 ++++++++++++++++++--------- lib/gitlab/gitaly_client/operation_service.rb | 32 +++++++++++++++- 2 files changed, 66 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index eab04bcac65..1468069a991 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -776,24 +776,21 @@ module Gitlab end def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - revert_tree_id = check_revert_content(commit, start_commit.sha) - raise CreateTreeError unless revert_tree_id - - committer = user_to_committer(user) + gitaly_migrate(:revert) do |is_enabled| + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - create_commit(message: message, - author: committer, - committer: committer, - tree: revert_tree_id, - parents: [start_commit.sha]) + if is_enabled + gitaly_operations_client.user_revert(args) + else + rugged_revert(args) + end end end @@ -1769,6 +1766,28 @@ module Gitlab end end + def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + revert_tree_id = check_revert_content(commit, start_commit.sha) + raise CreateTreeError unless revert_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: committer, + committer: committer, + tree: revert_tree_id, + parents: [start_commit.sha]) + end + end + def gitaly_add_branch(branch_name, user, target) gitaly_operation_client.user_create_branch(branch_name, user, target) rescue GRPC::FailedPrecondition => ex diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 51de6a9a75d..400a4af363b 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -124,7 +124,31 @@ module Gitlab end def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - request = Gitaly::UserCherryPickRequest.new( + call_cherry_pick_or_revert(:cherry_pick, + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository) + end + + def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + call_cherry_pick_or_revert(:revert, + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository) + end + + private + + def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize + + request = request_class.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit: commit.to_gitaly_commit, @@ -137,11 +161,15 @@ module Gitlab response = GitalyClient.call( @repository.storage, :operation_service, - :user_cherry_pick, + :"user_#{rpc}", request, remote_storage: start_repository.storage ) + handle_cherry_pick_or_revert_response(response) + end + + def handle_cherry_pick_or_revert_response(response) if response.pre_receive_error.presence raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error elsif response.commit_error.presence -- cgit v1.2.1 From 49dd62ada1f88f51f426e97c0ece18ec71e39514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Tue, 28 Nov 2017 10:23:38 +0100 Subject: Migrate Gitlab::Git::Commit.shas_with_signatures - Added tests for Git::Commit.shas_with_signatures --- lib/gitlab/git/commit.rb | 16 +++++++++++----- lib/gitlab/gitaly_client/commit_service.rb | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index c85dcfa0475..8900e2d7afe 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -213,11 +213,17 @@ module Gitlab end def shas_with_signatures(repository, shas) - shas.select do |sha| - begin - Rugged::Commit.extract_signature(repository.rugged, sha) - rescue Rugged::OdbError - false + GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) + else + shas.select do |sha| + begin + Rugged::Commit.extract_signature(repository.rugged, sha) + rescue Rugged::OdbError + false + end + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 34807d280e5..7985f5b5457 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -250,6 +250,26 @@ module Gitlab consume_commits_response(response) end + def filter_shas_with_signatures(shas) + request = Gitaly::FilterShasWithSignaturesRequest.new(repository: @gitaly_repo) + + enum = Enumerator.new do |y| + shas.each_slice(20) do |revs| + request.shas = GitalyClient.encode_repeated(revs) + + y.yield request + + request = Gitaly::FilterShasWithSignaturesRequest.new + end + end + + response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum) + + response.flat_map do |msg| + msg.shas.map { |sha| EncodingHelper.encode!(sha) } + end + end + private def call_commit_diff(request_params, options = {}) -- cgit v1.2.1 From 02d97d46216b60568fb248f9aeb779528abd8e9c Mon Sep 17 00:00:00 2001 From: Brett Walker Date: Tue, 5 Dec 2017 16:43:47 +0000 Subject: add Gitlab::Database.replication_slots_supported? --- lib/gitlab/database.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index cd7b4c043da..96922e1a62f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -50,6 +50,10 @@ module Gitlab postgresql? && version.to_f >= 9.3 end + def self.replication_slots_supported? + postgresql? && version.to_f >= 9.4 + end + def self.nulls_last_order(field, direction = 'ASC') order = "#{field} #{direction}" -- cgit v1.2.1 From 869d08b581495161352a661ac29b20b3925deaf0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Dec 2017 12:26:20 -0800 Subject: Process normal paths in batch containing bad paths --- .../populate_untracked_uploads.rb | 29 +++++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ebb483c3cff..81e95e5832d 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -57,7 +57,7 @@ module Gitlab ].freeze def to_h - { + @upload_hash ||= { path: upload_path, uploader: uploader, model_type: model_type, @@ -156,8 +156,8 @@ module Gitlab return unless migrate? files = UntrackedFile.where(id: start_id..end_id) - insert_uploads_if_needed(files) - files.delete_all + processed_files = insert_uploads_if_needed(files) + processed_files.delete_all drop_temp_table_if_finished end @@ -169,9 +169,30 @@ module Gitlab end def insert_uploads_if_needed(files) - filtered_files = filter_existing_uploads(files) + filtered_files, error_files = filter_error_files(files) + filtered_files = filter_existing_uploads(filtered_files) filtered_files = filter_deleted_models(filtered_files) insert(filtered_files) + + processed_files = files.where.not(id: error_files.map(&:id)) + processed_files + end + + def filter_error_files(files) + files.partition do |file| + begin + file.to_h + true + rescue => e + msg = <<~MSG + Error parsing path "#{file.path}": + #{e.message} + #{e.backtrace.join("\n ")} + MSG + Rails.logger.error(msg) + false + end + end end def filter_existing_uploads(files) -- cgit v1.2.1 From 382a1ef45360b538a80fd5d34cde71f53522dc2a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 6 Dec 2017 15:11:38 +0100 Subject: Add invalid builds counter metric to stage seeds class --- lib/gitlab/ci/stage/seed.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index bc97aa63b02..6a3b2f6cc4a 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -39,6 +39,10 @@ module Gitlab pipeline.stages.create!(stage).tap do |stage| builds_attributes = builds.map do |attributes| attributes.merge(stage_id: stage.id) + + if attributes.fetch(:stage_id).nil? + invalid_builds_counter.increment(node: hostname) + end end pipeline.builds.create!(builds_attributes).each do |build| @@ -52,6 +56,15 @@ module Gitlab def protected_ref? @protected_ref ||= project.protected_for?(pipeline.ref) end + + def invalid_builds_counter + @counter ||= Gitlab::Metrics.counter(:invalid_builds_counter, + 'Builds without stage assigned counter') + end + + def hostname + @hostname ||= Socket.gethostname + end end end end -- cgit v1.2.1 From 5ccced63122fe2d21fff7d7298db1081776193b4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 6 Dec 2017 15:30:20 +0100 Subject: Move invalid builds counter out of the transaction --- lib/gitlab/ci/pipeline/chain/create.rb | 17 +++++++++++++++++ lib/gitlab/ci/stage/seed.rb | 13 ------------- 2 files changed, 17 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d5e17a123df..42ae1650437 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -17,11 +17,28 @@ module Gitlab end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") + ensure + pipeline.builds.find_each do |build| + next if build.stage_id.present? + + invalid_builds_counter.increment(node: hostname) + end end def break? !pipeline.persisted? end + + private + + def invalid_builds_counter + @counter ||= Gitlab::Metrics + .counter(:invalid_builds_counter, 'Invalid builds counter') + end + + def hostname + @hostname ||= Socket.gethostname + end end end end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index 6a3b2f6cc4a..bc97aa63b02 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -39,10 +39,6 @@ module Gitlab pipeline.stages.create!(stage).tap do |stage| builds_attributes = builds.map do |attributes| attributes.merge(stage_id: stage.id) - - if attributes.fetch(:stage_id).nil? - invalid_builds_counter.increment(node: hostname) - end end pipeline.builds.create!(builds_attributes).each do |build| @@ -56,15 +52,6 @@ module Gitlab def protected_ref? @protected_ref ||= project.protected_for?(pipeline.ref) end - - def invalid_builds_counter - @counter ||= Gitlab::Metrics.counter(:invalid_builds_counter, - 'Builds without stage assigned counter') - end - - def hostname - @hostname ||= Socket.gethostname - end end end end -- cgit v1.2.1 From d704f0c4a0af626aac48d5a74ab6f05f7233a496 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 6 Dec 2017 18:28:27 +0100 Subject: Prevent dups when using StringIO for binary reads --- lib/gitlab/gitaly_client.rb | 6 ++++++ lib/gitlab/gitaly_client/wiki_service.rb | 10 ++++------ 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f27cd800bdd..1fe938a39a8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -336,6 +336,12 @@ module Gitlab s.dup.force_encoding(Encoding::ASCII_8BIT) end + def self.binary_stringio(s) + io = StringIO.new(s || '') + io.set_encoding(Encoding::ASCII_8BIT) + io + end + def self.encode_repeated(a) Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index c8f065f5881..337d225d081 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -18,12 +18,11 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - strio = StringIO.new(content) + strio = GitalyClient.binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? - chunk = strio.read(MAX_MSG_SIZE) - request.content = GitalyClient.encode(chunk) + request.content = strio.read(MAX_MSG_SIZE) y.yield request @@ -46,12 +45,11 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - strio = StringIO.new(content) + strio = GitalyClient.binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? - chunk = strio.read(MAX_MSG_SIZE) - request.content = GitalyClient.encode(chunk) + request.content = strio.read(MAX_MSG_SIZE) y.yield request -- cgit v1.2.1 From 0c2fdb1c342fa5eb64bb0ed02091af3c06b37c0e Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Fri, 1 Dec 2017 10:21:05 +0100 Subject: Refactor banzai to support referencing from group context --- lib/banzai/cross_project_reference.rb | 2 +- lib/banzai/filter/abstract_reference_filter.rb | 98 +++++++++++++--------- lib/banzai/filter/epic_reference_filter.rb | 12 +++ lib/banzai/filter/issuable_reference_filter.rb | 31 +++++++ lib/banzai/filter/issue_reference_filter.rb | 32 ++----- lib/banzai/filter/label_reference_filter.rb | 4 +- .../filter/merge_request_reference_filter.rb | 37 ++------ lib/banzai/filter/milestone_reference_filter.rb | 2 +- lib/banzai/issuable_extractor.rb | 4 +- lib/banzai/reference_parser/epic_parser.rb | 12 +++ lib/banzai/reference_parser/issuable_parser.rb | 25 ++++++ lib/banzai/reference_parser/issue_parser.rb | 12 +-- .../reference_parser/merge_request_parser.rb | 24 +----- lib/gitlab/reference_extractor.rb | 2 +- 14 files changed, 164 insertions(+), 133 deletions(-) create mode 100644 lib/banzai/filter/epic_reference_filter.rb create mode 100644 lib/banzai/filter/issuable_reference_filter.rb create mode 100644 lib/banzai/reference_parser/epic_parser.rb create mode 100644 lib/banzai/reference_parser/issuable_parser.rb (limited to 'lib') diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index e2b57adf611..d8fb7705b2a 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -11,7 +11,7 @@ module Banzai # ref - String reference. # # Returns a Project, or nil if the reference can't be found - def project_from_ref(ref) + def parent_from_ref(ref) return context[:project] unless ref Project.find_by_full_path(ref) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 8975395aff1..e7e6a90b5fd 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -82,9 +82,9 @@ module Banzai end end - def project_from_ref_cached(ref) - cached_call(:banzai_project_refs, ref) do - project_from_ref(ref) + def from_ref_cached(ref) + cached_call("banzai_#{parent_type}_refs".to_sym, ref) do + parent_from_ref(ref) end end @@ -153,15 +153,20 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| - project_path = full_project_path(namespace_ref, project_ref) - project = project_from_ref_cached(project_path) + parent_path = if parent_type == :group + full_group_path(namespace_ref) + else + full_project_path(namespace_ref, project_ref) + end - if project + parent = from_ref_cached(parent_path) + + if parent object = if link_reference - find_object_from_link_cached(project, id) + find_object_from_link_cached(parent, id) else - find_object_cached(project, id) + find_object_cached(parent, id) end end @@ -169,13 +174,13 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, project, object, link: !!link_content) + data = data_attributes_for(link_content || match, parent, object, link: !!link_content) url = if matches.names.include?("url") && matches[:url] matches[:url] else - url_for_object_cached(object, project) + url_for_object_cached(object, parent) end content = link_content || object_link_text(object, matches) @@ -224,17 +229,24 @@ module Banzai # Returns a Hash containing all object references (e.g. issue IDs) per the # project they belong to. - def references_per_project - @references_per_project ||= begin + def references_per_parent + @references_per ||= {} + + @references_per[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } regex = Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) nodes.each do |node| node.to_html.scan(regex) do - project_path = full_project_path($~[:namespace], $~[:project]) + path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + symbol = $~[object_sym] - refs[project_path] << symbol if object_class.reference_valid?(symbol) + refs[path] << symbol if object_class.reference_valid?(symbol) end end @@ -244,35 +256,41 @@ module Banzai # Returns a Hash containing referenced projects grouped per their full # path. - def projects_per_reference - @projects_per_reference ||= begin + def parent_per_reference + @per_reference ||= {} + + @per_reference[parent_type] ||= begin refs = Set.new - references_per_project.each do |project_ref, _| - refs << project_ref + references_per_parent.each do |ref, _| + refs << ref end - find_projects_for_paths(refs.to_a).index_by(&:full_path) + find_for_paths(refs.to_a).index_by(&:full_path) end end - def projects_relation_for_paths(paths) - Project.where_full_path_in(paths).includes(:namespace) + def relation_for_paths(paths) + klass = parent_type.to_s.camelize.constantize + result = klass.where_full_path_in(paths) + return result if parent_type == :group + + result.includes(:namespace) if parent_type == :project end # Returns projects for the given paths. - def find_projects_for_paths(paths) + def find_for_paths(paths) if RequestStore.active? - cache = project_refs_cache + cache = refs_cache to_query = paths - cache.keys unless to_query.empty? - projects = projects_relation_for_paths(to_query) + records = relation_for_paths(to_query) found = [] - projects.each do |project| - ref = project.full_path - get_or_set_cache(cache, ref) { project } + records.each do |record| + ref = record.full_path + get_or_set_cache(cache, ref) { record } found << ref end @@ -284,33 +302,37 @@ module Banzai cache.slice(*paths).values.compact else - projects_relation_for_paths(paths) + relation_for_paths(paths) end end - def current_project_path - return unless project - - @current_project_path ||= project.full_path + def current_parent_path + @current_parent_path ||= parent&.full_path end def current_project_namespace_path - return unless project - - @current_project_namespace_path ||= project.namespace.full_path + @current_project_namespace_path ||= project&.namespace&.full_path end private def full_project_path(namespace, project_ref) - return current_project_path unless project_ref + return current_parent_path unless project_ref namespace_ref = namespace || current_project_namespace_path "#{namespace_ref}/#{project_ref}" end - def project_refs_cache - RequestStore[:banzai_project_refs] ||= {} + def refs_cache + RequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + end + + def parent_type + :project + end + + def parent + parent_type == :project ? project : group end end end diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb new file mode 100644 index 00000000000..265924abe24 --- /dev/null +++ b/lib/banzai/filter/epic_reference_filter.rb @@ -0,0 +1,12 @@ +module Banzai + module Filter + # The actual filter is implemented in the EE mixin + class EpicReferenceFilter < IssuableReferenceFilter + self.reference_type = :epic + + def self.object_class + Epic + end + end + end +end diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb new file mode 100644 index 00000000000..7addf09be73 --- /dev/null +++ b/lib/banzai/filter/issuable_reference_filter.rb @@ -0,0 +1,31 @@ +module Banzai + module Filter + class IssuableReferenceFilter < AbstractReferenceFilter + def records_per_parent + @records_per_project ||= {} + + @records_per_project[object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + parent_records(parent, record_ids).each do |record| + hash[parent][record.iid.to_i] = record + end + end + + hash + end + end + + def find_object(parent, iid) + records_per_parent[parent][iid] + end + + def parent_from_ref(ref) + parent_per_reference[ref || current_parent_path] + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index ce1ab977d3b..6877cae8c55 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -8,46 +8,24 @@ module Banzai # When external issues tracker like Jira is activated we should not # use issue reference pattern, but we should still be able # to reference issues from other GitLab projects. - class IssueReferenceFilter < AbstractReferenceFilter + class IssueReferenceFilter < IssuableReferenceFilter self.reference_type = :issue def self.object_class Issue end - def find_object(project, iid) - issues_per_project[project][iid] - end - def url_for_object(issue, project) IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path], internal: true) end - def project_from_ref(ref) - projects_per_reference[ref || current_project_path] - end - - # Returns a Hash containing the issues per Project instance. - def issues_per_project - @issues_per_project ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - projects_per_reference.each do |path, project| - issue_ids = references_per_project[path] - issues = project.issues.where(iid: issue_ids.to_a) - - issues.each do |issue| - hash[project][issue.iid.to_i] = issue - end - end - - hash - end - end - def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end + + def parent_records(parent, ids) + parent.issues.where(iid: ids.to_a) + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 5364984c9d3..d5360ad8f68 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -33,7 +33,7 @@ module Banzai end def find_label(project_ref, label_id, label_name) - project = project_from_ref(project_ref) + project = parent_from_ref(project_ref) return unless project label_params = label_params(label_id, label_name) @@ -66,7 +66,7 @@ module Banzai def object_link_text(object, matches) project_path = full_project_path(matches[:namespace], matches[:project]) - project_from_ref = project_from_ref_cached(project_path) + project_from_ref = from_ref_cached(project_path) reference = project_from_ref.to_human_reference(project) label_suffix = " in #{reference}" if reference.present? diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 0eab865ac04..b3cfa97d0e0 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -4,48 +4,19 @@ module Banzai # to merge requests that do not exist are ignored. # # This filter supports cross-project references. - class MergeRequestReferenceFilter < AbstractReferenceFilter + class MergeRequestReferenceFilter < IssuableReferenceFilter self.reference_type = :merge_request def self.object_class MergeRequest end - def find_object(project, iid) - merge_requests_per_project[project][iid] - end - def url_for_object(mr, project) h = Gitlab::Routing.url_helpers h.project_merge_request_url(project, mr, only_path: context[:only_path]) end - def project_from_ref(ref) - projects_per_reference[ref || current_project_path] - end - - # Returns a Hash containing the merge_requests per Project instance. - def merge_requests_per_project - @merge_requests_per_project ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - projects_per_reference.each do |path, project| - merge_request_ids = references_per_project[path] - - merge_requests = project.merge_requests - .where(iid: merge_request_ids.to_a) - .includes(target_project: :namespace) - - merge_requests.each do |merge_request| - hash[project][merge_request.iid.to_i] = merge_request - end - end - - hash - end - end - def object_link_text_extras(object, matches) extras = super @@ -61,6 +32,12 @@ module Banzai extras end + + def parent_records(parent, ids) + parent.merge_requests + .where(iid: ids.to_a) + .includes(target_project: :namespace) + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index bb5da310e09..2a6b0964ac5 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -38,7 +38,7 @@ module Banzai def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) project_path = full_project_path(namespace_ref, project_ref) - project = project_from_ref(project_path) + project = parent_from_ref(project_path) return unless project diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index cbabf9156de..49603d0b363 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -28,8 +28,8 @@ module Banzai issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) - issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( - merge_request_parser.merge_requests_for_nodes(nodes) + issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( + merge_request_parser.records_for_nodes(nodes) ) # The project for the issue/MR might be pending for deletion! diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb new file mode 100644 index 00000000000..08b8a4c9a0f --- /dev/null +++ b/lib/banzai/reference_parser/epic_parser.rb @@ -0,0 +1,12 @@ +module Banzai + module ReferenceParser + # The actual parser is implemented in the EE mixin + class EpicParser < IssuableParser + self.reference_type = :epic + + def records_for_nodes(_nodes) + {} + end + end + end +end diff --git a/lib/banzai/reference_parser/issuable_parser.rb b/lib/banzai/reference_parser/issuable_parser.rb new file mode 100644 index 00000000000..3953867eb83 --- /dev/null +++ b/lib/banzai/reference_parser/issuable_parser.rb @@ -0,0 +1,25 @@ +module Banzai + module ReferenceParser + class IssuableParser < BaseParser + def nodes_visible_to_user(user, nodes) + records = records_for_nodes(nodes) + + nodes.select do |node| + issuable = records[node] + + issuable && can_read_reference?(user, issuable) + end + end + + def referenced_by(nodes) + records = records_for_nodes(nodes) + + nodes.map { |node| records[node] }.compact.uniq + end + + def can_read_reference?(user, issuable) + can?(user, "read_#{issuable.class.to_s.underscore}".to_sym, issuable) + end + end + end +end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index e0a8ca653cb..38d4e3f3e44 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -1,10 +1,10 @@ module Banzai module ReferenceParser - class IssueParser < BaseParser + class IssueParser < IssuableParser self.reference_type = :issue def nodes_visible_to_user(user, nodes) - issues = issues_for_nodes(nodes) + issues = records_for_nodes(nodes) readable_issues = Ability .issues_readable_by_user(issues.values, user).to_set @@ -14,13 +14,7 @@ module Banzai end end - def referenced_by(nodes) - issues = issues_for_nodes(nodes) - - nodes.map { |node| issues[node] }.compact.uniq - end - - def issues_for_nodes(nodes) + def records_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, Issue.all.includes( diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 75cbc7fdac4..a370ff5b5b3 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -1,25 +1,9 @@ module Banzai module ReferenceParser - class MergeRequestParser < BaseParser + class MergeRequestParser < IssuableParser self.reference_type = :merge_request - def nodes_visible_to_user(user, nodes) - merge_requests = merge_requests_for_nodes(nodes) - - nodes.select do |node| - merge_request = merge_requests[node] - - merge_request && can?(user, :read_merge_request, merge_request.project) - end - end - - def referenced_by(nodes) - merge_requests = merge_requests_for_nodes(nodes) - - nodes.map { |node| merge_requests[node] }.compact.uniq - end - - def merge_requests_for_nodes(nodes) + def records_for_nodes(nodes) @merge_requests_for_nodes ||= grouped_objects_for_nodes( nodes, MergeRequest.includes( @@ -40,10 +24,6 @@ module Banzai self.class.data_attribute ) end - - def can_read_reference?(user, ref_project, node) - can?(user, :read_merge_request, ref_project) - end end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index bc836dcc08d..9ff82d628c0 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,7 +1,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user).freeze + REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user epic).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) -- cgit v1.2.1 From c9871e84e4b605922689aee5957d206516dca8e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Thu, 7 Dec 2017 08:44:55 +0000 Subject: The API isn't using the appropriate services for managing forks --- lib/api/projects.rb | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 14a4fc6f025..fa222bf2b1c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -367,15 +367,16 @@ module API post ":id/fork/:forked_from_id" do authenticated_as_admin! - forked_from_project = find_project!(params[:forked_from_id]) - not_found!("Source Project") unless forked_from_project + fork_from_project = find_project!(params[:forked_from_id]) - if user_project.forked_from_project.nil? - user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + not_found!("Source Project") unless fork_from_project - ::Projects::ForksCountService.new(forked_from_project).refresh_cache + result = ::Projects::ForkService.new(fork_from_project, current_user).execute(user_project) + + if result + present user_project.reload, with: Entities::Project else - render_api_error!("Project already forked", 409) + render_api_error!("Project already forked", 409) if user_project.forked? end end @@ -383,11 +384,11 @@ module API delete ":id/fork" do authorize! :remove_fork_project, user_project - if user_project.forked? - destroy_conditionally!(user_project.forked_project_link) - else - not_modified! + result = destroy_conditionally!(user_project) do + ::Projects::UnlinkForkService.new(user_project, current_user).execute end + + result ? status(204) : not_modified! end desc 'Share the project with a group' do -- cgit v1.2.1 From be12f3ed24b7c2d120e249d52179eb09bff7c8aa Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 7 Dec 2017 10:27:07 +0100 Subject: Update pipeline create chain Prometheus metric --- lib/gitlab/ci/pipeline/chain/create.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 42ae1650437..d19a2519803 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -18,9 +18,7 @@ module Gitlab rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") ensure - pipeline.builds.find_each do |build| - next if build.stage_id.present? - + if pipeline.builds.where(stage_id: nil).any? invalid_builds_counter.increment(node: hostname) end end @@ -33,7 +31,8 @@ module Gitlab def invalid_builds_counter @counter ||= Gitlab::Metrics - .counter(:invalid_builds_counter, 'Invalid builds counter') + .counter(:gitlab_ci_invalid_builds_total, + 'Invalid builds without stage assigned counter') end def hostname -- cgit v1.2.1 From f7c18ca31469b199c1a898cef583c9aae99f1375 Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Wed, 6 Dec 2017 12:36:11 +0100 Subject: Support uploads for groups --- lib/banzai/filter/upload_link_filter.rb | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 09844931be5..d64f9ac4eb6 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -8,7 +8,7 @@ module Banzai # class UploadLinkFilter < HTML::Pipeline::Filter def call - return doc unless project + return doc unless project || group doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el| process_link_attr el.attribute('href') @@ -28,13 +28,27 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, project.full_path, uri) + base_path = Gitlab.config.gitlab.url + + if group + urls = Gitlab::Routing.url_helpers + # we need to get last 2 parts of the uri which are secret and filename + uri_parts = uri.split(File::SEPARATOR) + file_path = urls.show_group_uploads_path(group, uri_parts[-2], uri_parts[-1]) + File.join(base_path, file_path) + else + File.join(base_path, project.full_path, uri) + end end def project context[:project] end + def group + context[:group] + end + # Ensure that a :project key exists in context # # Note that while the key might exist, its value could be nil! -- cgit v1.2.1 From 1d47ae1365e259406233764885891923bebc555c Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Wed, 22 Nov 2017 17:08:47 +0000 Subject: CE backport of ProtectedBranches API changes In EE we now allow individual users/groups to be set on POST, which required some refactoring. See https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3516 --- lib/api/protected_branches.rb | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index b5021e8a712..614822509f0 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -39,10 +39,10 @@ module API end params do requires :name, type: String, desc: 'The name of the protected branch' - optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER, + optional :push_access_level, type: Integer, values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to push (defaults: `40`, master access level)' - optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER, + optional :merge_access_level, type: Integer, values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to merge (defaults: `40`, master access level)' end @@ -52,15 +52,13 @@ module API conflict!("Protected branch '#{params[:name]}' already exists") end - protected_branch_params = { - name: params[:name], - push_access_levels_attributes: [{ access_level: params[:push_access_level] }], - merge_access_levels_attributes: [{ access_level: params[:merge_access_level] }] - } + # Replace with `declared(params)` after updating to grape v1.0.2 + # See https://github.com/ruby-grape/grape/pull/1710 + # and https://gitlab.com/gitlab-org/gitlab-ce/issues/40843 + declared_params = params.slice("name", "push_access_level", "merge_access_level", "allowed_to_push", "allowed_to_merge") - service_args = [user_project, current_user, protected_branch_params] - - protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute + api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params) + protected_branch = api_service.create if protected_branch.persisted? present protected_branch, with: Entities::ProtectedBranch, project: user_project -- cgit v1.2.1 From 3d8fbd12b8f234aa62f4b5ceed21076a7afbcd23 Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Mon, 20 Nov 2017 09:02:01 -0500 Subject: add support for commit (in mr) to reference filter --- lib/banzai/filter/commit_reference_filter.rb | 14 ++++++++++++-- lib/banzai/object_renderer.rb | 16 ++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 714e0319025..f4e0c3111f5 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -24,8 +24,18 @@ module Banzai def url_for_object(commit, project) h = Gitlab::Routing.url_helpers - h.project_commit_url(project, commit, - only_path: context[:only_path]) + noteable = context[:merge_request] || context[:noteable] + + if noteable.is_a?(MergeRequest) && + noteable.all_commit_shas.include?(commit.id) + + # the internal shas are in the context? + # why not preload in the object?, just make sure we have the same ref + # in all the rendering + h.diffs_project_merge_request_url(project, noteable, commit_id: commit.id) + else + h.project_commit_url(project, commit, only_path: context[:only_path]) + end end def object_link_text_extras(object, matches) diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index ecb3affbba5..29c4e60f70c 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -18,10 +18,10 @@ module Banzai # project - A Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. # context - A Hash containing extra attributes to use during redaction - def initialize(project, user = nil, redaction_context = {}) + def initialize(project, user = nil, context = {}) @project = project @user = user - @redaction_context = redaction_context + @context = base_context.merge(context) end # Renders and redacts an Array of objects. @@ -48,7 +48,8 @@ module Banzai pipeline = HTML::Pipeline.new([]) objects.map do |object| - pipeline.to_document(Banzai.render_field(object, attribute)) + context = context_for(object, attribute) + pipeline.to_document(Banzai.render_field(object, attribute, context)) end end @@ -73,20 +74,19 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - base_context.merge(object.banzai_render_context(attribute)) + @context.merge(object.banzai_render_context(attribute)) end def base_context - @base_context ||= @redaction_context.merge( + { current_user: user, project: project, skip_redaction: true - ) + } end def save_options - return {} unless base_context[:xhtml] - + return {} unless @context[:xhtml] { save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML } end end -- cgit v1.2.1 From cb6f51ec9b2006f1040cca94119135c92e9a4cd1 Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Wed, 22 Nov 2017 09:48:09 -0500 Subject: add support for the commit reference filter --- lib/banzai/filter/commit_reference_filter.rb | 38 +++++++++++++++++++++------- lib/banzai/object_renderer.rb | 13 +++++----- lib/gitlab/diff/diff_refs.rb | 22 +++------------- lib/gitlab/git.rb | 12 +++++++++ lib/gitlab/git/commit.rb | 1 + 5 files changed, 51 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index f4e0c3111f5..c202d71072e 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -22,19 +22,29 @@ module Banzai end end + def referenced_merge_request_commit_shas + return [] unless noteable.is_a?(MergeRequest) + + @referenced_merge_request_commit_shas ||= begin + referenced_shas = references_per_project.values.reduce(:|).to_a + noteable.all_commit_shas.select do |sha| + referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) } + end + end + end + def url_for_object(commit, project) h = Gitlab::Routing.url_helpers - noteable = context[:merge_request] || context[:noteable] - - if noteable.is_a?(MergeRequest) && - noteable.all_commit_shas.include?(commit.id) - # the internal shas are in the context? - # why not preload in the object?, just make sure we have the same ref - # in all the rendering - h.diffs_project_merge_request_url(project, noteable, commit_id: commit.id) + if referenced_merge_request_commit_shas.include?(commit.id) + h.diffs_project_merge_request_url(project, + noteable, + commit_id: commit.id, + only_path: only_path?) else - h.project_commit_url(project, commit, only_path: context[:only_path]) + h.project_commit_url(project, + commit, + only_path: only_path?) end end @@ -48,6 +58,16 @@ module Banzai extras end + + private + + def noteable + context[:noteable] + end + + def only_path? + context[:only_path] + end end end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 29c4e60f70c..0bf9a8d66bc 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -17,11 +17,11 @@ module Banzai # project - A Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. - # context - A Hash containing extra attributes to use during redaction - def initialize(project, user = nil, context = {}) + # redaction_context - A Hash containing extra attributes to use during redaction + def initialize(project, user = nil, redaction_context = {}) @project = project @user = user - @context = base_context.merge(context) + @redaction_context = base_context.merge(redaction_context) end # Renders and redacts an Array of objects. @@ -48,8 +48,7 @@ module Banzai pipeline = HTML::Pipeline.new([]) objects.map do |object| - context = context_for(object, attribute) - pipeline.to_document(Banzai.render_field(object, attribute, context)) + pipeline.to_document(Banzai.render_field(object, attribute)) end end @@ -74,7 +73,7 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - @context.merge(object.banzai_render_context(attribute)) + @redaction_context.merge(object.banzai_render_context(attribute)) end def base_context @@ -86,7 +85,7 @@ module Banzai end def save_options - return {} unless @context[:xhtml] + return {} unless @redaction_context[:xhtml] { save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML } end end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index c98eefbce25..88e0db830f6 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -13,9 +13,9 @@ module Gitlab def ==(other) other.is_a?(self.class) && - shas_equal?(base_sha, other.base_sha) && - shas_equal?(start_sha, other.start_sha) && - shas_equal?(head_sha, other.head_sha) + Git.shas_eql?(base_sha, other.base_sha) && + Git.shas_eql?(start_sha, other.start_sha) && + Git.shas_eql?(head_sha, other.head_sha) end alias_method :eql?, :== @@ -47,22 +47,6 @@ module Gitlab CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) end end - - private - - def shas_equal?(sha1, sha2) - return true if sha1 == sha2 - return false if sha1.nil? || sha2.nil? - return false unless sha1.class == sha2.class - - length = [sha1.length, sha2.length].min - - # If either of the shas is below the minimum length, we cannot be sure - # that they actually refer to the same commit because of hash collision. - return false if length < Commit::MIN_SHA_LENGTH - - sha1[0, length] == sha2[0, length] - end end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 1f31cdbc96d..1f7c35cafaa 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -70,6 +70,18 @@ module Gitlab def diff_line_code(file_path, new_line_position, old_line_position) "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" end + + def shas_eql?(sha1, sha2) + return false if sha1.nil? || sha2.nil? + return false unless sha1.class == sha2.class + + # If either of the shas is below the minimum length, we cannot be sure + # that they actually refer to the same commit because of hash collision. + length = [sha1.length, sha2.length].min + return false if length < Gitlab::Git::Commit::MIN_SHA_LENGTH + + sha1[0, length] == sha2[0, length] + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 8900e2d7afe..e90b158fb34 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -6,6 +6,7 @@ module Gitlab attr_accessor :raw_commit, :head + MIN_SHA_LENGTH = 7 SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, -- cgit v1.2.1 From 360b94ceba146935a40b02f39ed3d833eaea134a Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Fri, 1 Dec 2017 14:08:30 -0500 Subject: adding view and feature specs --- lib/banzai/object_renderer.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 0bf9a8d66bc..2691be81623 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -86,6 +86,7 @@ module Banzai def save_options return {} unless @redaction_context[:xhtml] + { save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML } end end -- cgit v1.2.1 From 03ac8d5d0b8331a2ecfae05137e02c78428fa899 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 7 Dec 2017 15:33:30 +0000 Subject: Remove Rugged::Repository#empty? --- lib/backup/repository.rb | 9 +++------ lib/gitlab/git/operation_service.rb | 2 +- lib/gitlab/git/remote_repository.rb | 6 ++++-- lib/gitlab/git/repository.rb | 32 ++++++++++---------------------- lib/gitlab/gitaly_client.rb | 2 +- 5 files changed, 19 insertions(+), 32 deletions(-) (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index b6d273b98c2..2a04c03919d 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -193,12 +193,9 @@ module Backup end def empty_repo?(project_or_wiki) - project_or_wiki.repository.expire_exists_cache # protect backups from stale cache - project_or_wiki.repository.empty_repo? - rescue => e - progress.puts "Ignoring repository error and continuing backing up project: #{display_repo_path(project_or_wiki)} - #{e.message}".color(:orange) - - false + # Protect against stale caches + project_or_wiki.repository.expire_emptiness_caches + project_or_wiki.repository.empty? end def repository_storage_paths_args diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index e36d5410431..7e8fe173056 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -83,7 +83,7 @@ module Gitlab Gitlab::Git.check_namespace!(start_repository) start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - start_branch_name = nil if start_repository.empty_repo? + start_branch_name = nil if start_repository.empty? if start_branch_name && !start_repository.branch_exists?(start_branch_name) raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb index 3685aa20669..6bd6e58feeb 100644 --- a/lib/gitlab/git/remote_repository.rb +++ b/lib/gitlab/git/remote_repository.rb @@ -24,10 +24,12 @@ module Gitlab @path = repository.path end - def empty_repo? + def empty? # We will override this implementation in gitaly-ruby because we cannot # use '@repository' there. - @repository.empty_repo? + # + # Caches and memoization used on the Rails side + !@repository.exists? || @repository.empty? end def commit_id(revision) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1468069a991..91dd2fbbdbc 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -75,9 +75,6 @@ module Gitlab @attributes = Gitlab::Git::Attributes.new(path) end - delegate :empty?, - to: :rugged - def ==(other) path == other.path end @@ -206,6 +203,13 @@ module Gitlab end end + # Git repository can contains some hidden refs like: + # /refs/notes/* + # /refs/git-as-svn/* + # /refs/pulls/* + # This refs by default not visible in project page and not cloned to client side. + alias_method :has_visible_content?, :has_local_branches? + def has_local_branches_rugged? rugged.branches.each(:local).any? do |ref| begin @@ -1004,7 +1008,7 @@ module Gitlab Gitlab::Git.check_namespace!(start_repository) start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - return yield nil if start_repository.empty_repo? + return yield nil if start_repository.empty? if start_repository.same_repository?(self) yield commit(start_branch_name) @@ -1120,24 +1124,8 @@ module Gitlab Gitlab::Git::Commit.find(self, ref) end - # Refactoring aid; allows us to copy code from app/models/repository.rb - def empty_repo? - !exists? || !has_visible_content? - end - - # - # Git repository can contains some hidden refs like: - # /refs/notes/* - # /refs/git-as-svn/* - # /refs/pulls/* - # This refs by default not visible in project page and not cloned to client side. - # - # This method return true if repository contains some content visible in project page. - # - def has_visible_content? - return @has_visible_content if defined?(@has_visible_content) - - @has_visible_content = has_local_branches? + def empty? + !has_visible_content? end # Like all public `Gitlab::Git::Repository` methods, this method is part diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 1fe938a39a8..b753ac46291 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -27,7 +27,7 @@ module Gitlab end SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - MAXIMUM_GITALY_CALLS = 30 + MAXIMUM_GITALY_CALLS = 35 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new -- cgit v1.2.1 From daf9357aa92283e6cdd0d1e0cade65c8e2294540 Mon Sep 17 00:00:00 2001 From: "micael.bergeron" Date: Thu, 7 Dec 2017 10:35:56 -0500 Subject: fix the missing reference to #references_per_project --- lib/banzai/filter/commit_reference_filter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index c202d71072e..eedb95197aa 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -26,7 +26,7 @@ module Banzai return [] unless noteable.is_a?(MergeRequest) @referenced_merge_request_commit_shas ||= begin - referenced_shas = references_per_project.values.reduce(:|).to_a + referenced_shas = references_per_parent.values.reduce(:|).to_a noteable.all_commit_shas.select do |sha| referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) } end -- cgit v1.2.1 From 7af56500a1e1b6437a2fa27988a5f11a4eb54391 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 7 Dec 2017 17:13:40 +0100 Subject: refactor code to match EE changes --- lib/gitlab/git_access.rb | 6 +++++- lib/gitlab/git_access_wiki.rb | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 8998c4b1a83..9d7d921bb9c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -166,7 +166,7 @@ module Gitlab end if Gitlab::Database.read_only? - raise UnauthorizedError, ERROR_MESSAGES[:cannot_push_to_read_only] + raise UnauthorizedError, push_to_read_only_message end if deploy_key @@ -280,5 +280,9 @@ module Gitlab UserAccess.new(user, project: project) end end + + def push_to_read_only_message + ERROR_MESSAGES[:cannot_push_to_read_only] + end end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 98f1f45b338..1c9477e84b2 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -19,10 +19,14 @@ module Gitlab end if Gitlab::Database.read_only? - raise UnauthorizedError, ERROR_MESSAGES[:read_only] + raise UnauthorizedError, push_to_read_only_message end true end + + def push_to_read_only_message + ERROR_MESSAGES[:read_only] + end end end -- cgit v1.2.1 From ee22a47d629fb13a52100921761f833acd80dbd9 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 7 Dec 2017 17:47:23 +0100 Subject: Update prometheus-client-mmap gem to highly optimized version + change string concatenation to help with GC pressure. + fix metric producing incompatible label sets --- lib/gitlab/metrics/samplers/ruby_sampler.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 436a9e9550d..f4901be9581 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -32,7 +32,7 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', {}) + metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) GC.stat.keys.each do |key| metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) @@ -100,9 +100,9 @@ module Gitlab worker_no = ::Prometheus::Client::Support::Unicorn.worker_id if worker_no - { unicorn: worker_no } + { worker: worker_no } else - { unicorn: 'master' } + { worker: 'master' } end end end -- cgit v1.2.1 From e6ac6734c2f636d3d063718a95ba1169e299b51f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 4 Dec 2017 20:18:45 -0600 Subject: Use relative _path helper URLs in the GitLab UI Fix https://gitlab.com/gitlab-org/gitlab-ce/issues/40825 --- lib/api/entities.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 62ee20bf7de..d96e7f2770f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -16,10 +16,13 @@ module API class UserBasic < UserSafe expose :state + expose :avatar_url do |user, options| user.avatar_url(only_path: false) end + expose :avatar_path, if: ->(user, options) { options.fetch(:only_path, false) && user.avatar_path } + expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) end -- cgit v1.2.1 From b7a5125f02865637ffbd4d7f8a623c994b861f0e Mon Sep 17 00:00:00 2001 From: Martin Nowak Date: Wed, 6 Dec 2017 19:10:53 +0100 Subject: fix #39233 - 500 in merge request - handle unchanged empty lines in inline diff --- lib/gitlab/diff/inline_diff.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 2d7b57120a6..54783a07919 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -70,7 +70,7 @@ module Gitlab def find_changed_line_pairs(lines) # Prefixes of all diff lines, indicating their types # For example: `" - + -+ ---+++ --+ -++"` - line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ') + line_prefixes = lines.each_with_object("") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') changed_line_pairs = [] line_prefixes.scan(LINE_PAIRS_PATTERN) do -- cgit v1.2.1 From 41b0c0e9be7a34d1255326904a234ae142d6c205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20Carlb=C3=A4cker?= Date: Fri, 8 Dec 2017 00:27:11 +0000 Subject: Migrate Git::Repository#fsck to Gitaly --- lib/gitlab/git/repository.rb | 18 ++++++++++++++++-- lib/gitlab/gitaly_client/repository_service.rb | 11 +++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 91dd2fbbdbc..73889328f36 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1160,9 +1160,15 @@ module Gitlab end def fsck - output, status = run_git(%W[--git-dir=#{path} fsck], nice: true) + gitaly_migrate(:git_fsck) do |is_enabled| + msg, status = if is_enabled + gitaly_fsck + else + shell_fsck + end - raise GitError.new("Could not fsck repository:\n#{output}") unless status.zero? + raise GitError.new("Could not fsck repository: #{msg}") unless status.zero? + end end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) @@ -1310,6 +1316,14 @@ module Gitlab File.write(File.join(worktree_info_path, 'sparse-checkout'), files) end + def gitaly_fsck + gitaly_repository_client.fsck + end + + def shell_fsck + run_git(%W[--git-dir=#{path} fsck], nice: true) + end + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) with_repo_branch_commit(source_repository, source_branch) do |commit| if commit diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index b9e606592d7..a477d618f63 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -87,6 +87,17 @@ module Gitlab response.result end + + def fsck + request = Gitaly::FsckRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :fsck, request) + + if response.error.empty? + return "", 0 + else + return response.error.b, 1 + end + end end end end -- cgit v1.2.1 From f1ae1e39ce6b7578c5697c977bc3b52b119301ab Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 13 Nov 2017 16:52:07 +0100 Subject: Move the circuitbreaker check out in a separate process Moving the check out of the general requests, makes sure we don't have any slowdown in the regular requests. To keep the process performing this checks small, the check is still performed inside a unicorn. But that is called from a process running on the same server. Because the checks are now done outside normal request, we can have a simpler failure strategy: The check is now performed in the background every `circuitbreaker_check_interval`. Failures are logged in redis. The failures are reset when the check succeeds. Per check we will try `circuitbreaker_access_retries` times within `circuitbreaker_storage_timeout` seconds. When the number of failures exceeds `circuitbreaker_failure_count_threshold`, we will block access to the storage. After `failure_reset_time` of no checks, we will clear the stored failures. This could happen when the process that performs the checks is not running. --- lib/api/circuit_breakers.rb | 2 +- lib/gitlab/git/storage/checker.rb | 98 +++++++++++++++++++ lib/gitlab/git/storage/circuit_breaker.rb | 106 +-------------------- lib/gitlab/git/storage/circuit_breaker_settings.rb | 12 +-- lib/gitlab/git/storage/failure_info.rb | 39 ++++++++ lib/gitlab/git/storage/null_circuit_breaker.rb | 22 +++-- lib/gitlab/storage_check.rb | 11 +++ lib/gitlab/storage_check/cli.rb | 69 ++++++++++++++ lib/gitlab/storage_check/gitlab_caller.rb | 39 ++++++++ lib/gitlab/storage_check/option_parser.rb | 39 ++++++++ lib/gitlab/storage_check/response.rb | 77 +++++++++++++++ 11 files changed, 396 insertions(+), 118 deletions(-) create mode 100644 lib/gitlab/git/storage/checker.rb create mode 100644 lib/gitlab/git/storage/failure_info.rb create mode 100644 lib/gitlab/storage_check.rb create mode 100644 lib/gitlab/storage_check/cli.rb create mode 100644 lib/gitlab/storage_check/gitlab_caller.rb create mode 100644 lib/gitlab/storage_check/option_parser.rb create mode 100644 lib/gitlab/storage_check/response.rb (limited to 'lib') diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb index 118883f5ea5..598c76f6168 100644 --- a/lib/api/circuit_breakers.rb +++ b/lib/api/circuit_breakers.rb @@ -41,7 +41,7 @@ module API detail 'This feature was introduced in GitLab 9.5' end delete do - Gitlab::Git::Storage::CircuitBreaker.reset_all! + Gitlab::Git::Storage::FailureInfo.reset_all! end end end diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb new file mode 100644 index 00000000000..de63cb4b40c --- /dev/null +++ b/lib/gitlab/git/storage/checker.rb @@ -0,0 +1,98 @@ +module Gitlab + module Git + module Storage + class Checker + include CircuitBreakerSettings + + attr_reader :storage_path, :storage, :hostname, :logger + + def self.check_all(logger = Rails.logger) + threads = Gitlab.config.repositories.storages.keys.map do |storage_name| + Thread.new do + Thread.current[:result] = new(storage_name, logger).check_with_lease + end + end + + threads.map do |thread| + thread.join + thread[:result] + end + end + + def initialize(storage, logger = Rails.logger) + @storage = storage + config = Gitlab.config.repositories.storages[@storage] + @storage_path = config['path'] + @logger = logger + + @hostname = Gitlab::Environment.hostname + end + + def check_with_lease + lease_key = "storage_check:#{cache_key}" + lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout) + result = { storage: storage, success: nil } + + if uuid = lease.try_obtain + result[:success] = check + + Gitlab::ExclusiveLease.cancel(lease_key, uuid) + else + logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running") + end + + result + end + + def check + if Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries) + track_storage_accessible + true + else + track_storage_inaccessible + logger.error("#{hostname}: #{storage}: Not accessible.") + false + end + end + + private + + def track_storage_inaccessible + first_failure = current_failure_info.first_failure || Time.now + last_failure = Time.now + + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :first_failure, first_failure.to_i) + redis.hset(cache_key, :last_failure, last_failure.to_i) + redis.hincrby(cache_key, :failure_count, 1) + redis.expire(cache_key, failure_reset_time) + maintain_known_keys(redis) + end + end + end + + def track_storage_accessible + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :first_failure, nil) + redis.hset(cache_key, :last_failure, nil) + redis.hset(cache_key, :failure_count, 0) + maintain_known_keys(redis) + end + end + end + + def maintain_known_keys(redis) + expire_time = Time.now.to_i + failure_reset_time + redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) + redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) + end + + def current_failure_info + FailureInfo.load(cache_key) + end + end + end + end +end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 4328c0ea29b..898bb1b65be 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -4,22 +4,11 @@ module Gitlab class CircuitBreaker include CircuitBreakerSettings - FailureInfo = Struct.new(:last_failure, :failure_count) - attr_reader :storage, - :hostname, - :storage_path - - delegate :last_failure, :failure_count, to: :failure_info - - def self.reset_all! - Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - redis.del(*all_storage_keys) unless all_storage_keys.empty? - end + :hostname - RequestStore.delete(:circuitbreaker_cache) - end + delegate :last_failure, :failure_count, :no_failures?, + to: :failure_info def self.for_storage(storage) cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do @@ -46,9 +35,6 @@ module Gitlab def initialize(storage, hostname) @storage = storage @hostname = hostname - - config = Gitlab.config.repositories.storages[@storage] - @storage_path = config['path'] end def perform @@ -65,15 +51,6 @@ module Gitlab failure_count > failure_count_threshold end - def backing_off? - return false if no_failures? - - recent_failure = last_failure > failure_wait_time.seconds.ago - too_many_failures = failure_count > backoff_threshold - - recent_failure && too_many_failures - end - private # The circuitbreaker can be enabled for the entire fleet using a Feature @@ -86,88 +63,13 @@ module Gitlab end def failure_info - @failure_info ||= get_failure_info - end - - # Memoizing the `storage_available` call means we only do it once per - # request when the storage is available. - # - # When the storage appears not available, and the memoized value is `false` - # we might want to try again. - def storage_available? - return @storage_available if @storage_available - - if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck - .storage_available?(storage_path, storage_timeout, access_retries) - track_storage_accessible - else - track_storage_inaccessible - end - - @storage_available + @failure_info ||= FailureInfo.load(cache_key) end def check_storage_accessible! if circuit_broken? raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) end - - if backing_off? - raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time) - end - - unless storage_available? - raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time) - end - end - - def no_failures? - last_failure.blank? && failure_count == 0 - end - - def track_storage_inaccessible - @failure_info = FailureInfo.new(Time.now, failure_count + 1) - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :last_failure, last_failure.to_i) - redis.hincrby(cache_key, :failure_count, 1) - redis.expire(cache_key, failure_reset_time) - maintain_known_keys(redis) - end - end - end - - def track_storage_accessible - @failure_info = FailureInfo.new(nil, 0) - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :last_failure, nil) - redis.hset(cache_key, :failure_count, 0) - maintain_known_keys(redis) - end - end - end - - def maintain_known_keys(redis) - expire_time = Time.now.to_i + failure_reset_time - redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) - redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) - end - - def get_failure_info - last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| - redis.hmget(cache_key, :last_failure, :failure_count) - end - - last_failure = Time.at(last_failure.to_i) if last_failure.present? - - FailureInfo.new(last_failure, failure_count.to_i) - end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end end end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb index 257fe8cd8f0..c9e225f187d 100644 --- a/lib/gitlab/git/storage/circuit_breaker_settings.rb +++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb @@ -6,10 +6,6 @@ module Gitlab application_settings.circuitbreaker_failure_count_threshold end - def failure_wait_time - application_settings.circuitbreaker_failure_wait_time - end - def failure_reset_time application_settings.circuitbreaker_failure_reset_time end @@ -22,8 +18,12 @@ module Gitlab application_settings.circuitbreaker_access_retries end - def backoff_threshold - application_settings.circuitbreaker_backoff_threshold + def check_interval + application_settings.circuitbreaker_check_interval + end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end private diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb new file mode 100644 index 00000000000..387279c110d --- /dev/null +++ b/lib/gitlab/git/storage/failure_info.rb @@ -0,0 +1,39 @@ +module Gitlab + module Git + module Storage + class FailureInfo + attr_accessor :first_failure, :last_failure, :failure_count + + def self.reset_all! + Gitlab::Git::Storage.redis.with do |redis| + all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) + redis.del(*all_storage_keys) unless all_storage_keys.empty? + end + + RequestStore.delete(:circuitbreaker_cache) + end + + def self.load(cache_key) + first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| + redis.hmget(cache_key, :first_failure, :last_failure, :failure_count) + end + + last_failure = Time.at(last_failure.to_i) if last_failure.present? + first_failure = Time.at(first_failure.to_i) if first_failure.present? + + new(first_failure, last_failure, failure_count.to_i) + end + + def initialize(first_failure, last_failure, failure_count) + @first_failure = first_failure + @last_failure = last_failure + @failure_count = failure_count + end + + def no_failures? + first_failure.blank? && last_failure.blank? && failure_count == 0 + end + end + end + end +end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb index a12d52d295f..261c936c689 100644 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -11,6 +11,9 @@ module Gitlab # These will always have nil values attr_reader :storage_path + delegate :last_failure, :failure_count, :no_failures?, + to: :failure_info + def initialize(storage, hostname, error: nil) @storage = storage @hostname = hostname @@ -29,16 +32,17 @@ module Gitlab false end - def last_failure - circuit_broken? ? Time.now : nil - end - - def failure_count - circuit_broken? ? failure_count_threshold : 0 - end - def failure_info - Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count) + @failure_info ||= + if circuit_broken? + Gitlab::Git::Storage::FailureInfo.new(Time.now, + Time.now, + failure_count_threshold) + else + Gitlab::Git::Storage::FailureInfo.new(nil, + nil, + 0) + end end end end diff --git a/lib/gitlab/storage_check.rb b/lib/gitlab/storage_check.rb new file mode 100644 index 00000000000..fe81513c9ec --- /dev/null +++ b/lib/gitlab/storage_check.rb @@ -0,0 +1,11 @@ +require_relative 'storage_check/cli' +require_relative 'storage_check/gitlab_caller' +require_relative 'storage_check/option_parser' +require_relative 'storage_check/response' + +module Gitlab + module StorageCheck + ENDPOINT = '/-/storage_check'.freeze + Options = Struct.new(:target, :token, :interval, :dryrun) + end +end diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb new file mode 100644 index 00000000000..04bf1bf1d26 --- /dev/null +++ b/lib/gitlab/storage_check/cli.rb @@ -0,0 +1,69 @@ +module Gitlab + module StorageCheck + class CLI + def self.start!(args) + runner = new(Gitlab::StorageCheck::OptionParser.parse!(args)) + runner.start_loop + end + + attr_reader :logger, :options + + def initialize(options) + @options = options + @logger = Logger.new(STDOUT) + end + + def start_loop + logger.info "Checking #{options.target} every #{options.interval} seconds" + + if options.dryrun + logger.info "Dryrun, exiting..." + return + end + + begin + loop do + response = GitlabCaller.new(options).call! + log_response(response) + update_settings(response) + + sleep options.interval + end + rescue Interrupt + logger.info "Ending storage-check" + end + end + + def update_settings(response) + previous_interval = options.interval + + if response.valid? + options.interval = response.check_interval || previous_interval + end + + if previous_interval != options.interval + logger.info "Interval changed: #{options.interval} seconds" + end + end + + def log_response(response) + unless response.valid? + return logger.error("Invalid response checking nfs storage: #{response.http_response.inspect}") + end + + if response.responsive_shards.any? + logger.debug("Responsive shards: #{response.responsive_shards.join(', ')}") + end + + warnings = [] + if response.skipped_shards.any? + warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" + end + if response.failing_shards.any? + warnings << "Failing shards: #{response.failing_shards.join(', ')}" + end + logger.warn(warnings.join(' - ')) if warnings.any? + end + end + end +end diff --git a/lib/gitlab/storage_check/gitlab_caller.rb b/lib/gitlab/storage_check/gitlab_caller.rb new file mode 100644 index 00000000000..44952b68844 --- /dev/null +++ b/lib/gitlab/storage_check/gitlab_caller.rb @@ -0,0 +1,39 @@ +require 'excon' + +module Gitlab + module StorageCheck + class GitlabCaller + def initialize(options) + @options = options + end + + def call! + Gitlab::StorageCheck::Response.new(get_response) + rescue Errno::ECONNREFUSED, Excon::Error + # Server not ready, treated as invalid response. + Gitlab::StorageCheck::Response.new(nil) + end + + def get_response + scheme, *other_parts = URI.split(@options.target) + socket_path = if scheme == 'unix' + other_parts.compact.join + end + + connection = Excon.new(@options.target, socket: socket_path) + connection.post(path: Gitlab::StorageCheck::ENDPOINT, + headers: headers) + end + + def headers + @headers ||= begin + headers = {} + headers['Content-Type'] = headers['Accept'] = 'application/json' + headers['TOKEN'] = @options.token if @options.token + + headers + end + end + end + end +end diff --git a/lib/gitlab/storage_check/option_parser.rb b/lib/gitlab/storage_check/option_parser.rb new file mode 100644 index 00000000000..66ed7906f97 --- /dev/null +++ b/lib/gitlab/storage_check/option_parser.rb @@ -0,0 +1,39 @@ +module Gitlab + module StorageCheck + class OptionParser + def self.parse!(args) + # Start out with some defaults + options = Gitlab::StorageCheck::Options.new(nil, nil, 1, false) + + parser = ::OptionParser.new do |opts| + opts.banner = "Usage: bin/storage_check [options]" + + opts.on('-t=string', '--target string', 'URL or socket to trigger storage check') do |value| + options.target = value + end + + opts.on('-T=string', '--token string', 'Health token to use') { |value| options.token = value } + + opts.on('-i=n', '--interval n', ::OptionParser::DecimalInteger, 'Seconds between checks') do |value| + options.interval = value + end + + opts.on('-d', '--dryrun', "Output what will be performed, but don't start the process") do |value| + options.dryrun = value + end + end + parser.parse!(args) + + unless options.target + raise ::OptionParser::InvalidArgument.new('Provide a URI to provide checks') + end + + if URI.parse(options.target).scheme.nil? + raise ::OptionParser::InvalidArgument.new('Add the scheme to the target, `unix://`, `https://` or `http://` are supported') + end + + options + end + end + end +end diff --git a/lib/gitlab/storage_check/response.rb b/lib/gitlab/storage_check/response.rb new file mode 100644 index 00000000000..326ab236e3e --- /dev/null +++ b/lib/gitlab/storage_check/response.rb @@ -0,0 +1,77 @@ +require 'json' + +module Gitlab + module StorageCheck + class Response + attr_reader :http_response + + def initialize(http_response) + @http_response = http_response + end + + def valid? + @http_response && (200...299).cover?(@http_response.status) && + @http_response.headers['Content-Type'].include?('application/json') && + parsed_response + end + + def check_interval + return nil unless parsed_response + + parsed_response['check_interval'] + end + + def responsive_shards + divided_results[:responsive_shards] + end + + def skipped_shards + divided_results[:skipped_shards] + end + + def failing_shards + divided_results[:failing_shards] + end + + private + + def results + return [] unless parsed_response + + parsed_response['results'] + end + + def divided_results + return @divided_results if @divided_results + + @divided_results = {} + @divided_results[:responsive_shards] = [] + @divided_results[:skipped_shards] = [] + @divided_results[:failing_shards] = [] + + results.each do |info| + name = info['storage'] + + case info['success'] + when true + @divided_results[:responsive_shards] << name + when false + @divided_results[:failing_shards] << name + else + @divided_results[:skipped_shards] << name + end + end + + @divided_results + end + + def parsed_response + return @parsed_response if defined?(@parsed_response) + + @parsed_response = JSON.parse(@http_response.body) + rescue JSON::JSONError + @parsed_response = nil + end + end + end +end -- cgit v1.2.1