diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2017-11-30 15:20:00 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2017-11-30 15:20:00 +0800 |
| commit | 85be6d83be4632c76760e373da131a90afb093b9 (patch) | |
| tree | 7ed7312dd8ad6e8e0ebd30b78774261c30c55d4e /lib | |
| parent | 689658456f706be7278fbf50fcde9c7f43cd0655 (diff) | |
| parent | f7254a4060b30e3134c6cf932eaba0fc8e249e9a (diff) | |
| download | gitlab-ce-85be6d83be4632c76760e373da131a90afb093b9.tar.gz | |
Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules
* upstream/master: (170 commits)
support ordering of project notes in notes api
Redirect to an already forked project if it exists
Reschedule the migration to populate fork networks
Create fork networks for forks for which the source was deleted.
Fix item name and namespace text overflow in Projects dropdown
Minor backport from EE
fix link that was linking to `html` instead of `md`
Backport epic tasklist
Add timeouts for Gitaly calls
SSHUploadPack over Gitaly is now OptOut
fix icon colors in commit list
Fix star icon color/stroke
Backport border inline edit
Add checkboxes to automatically run AutoDevops pipeline
BE for automatic pipeline when enabling Auto DevOps
I am certainly weary of debugging sidekiq but I don't think that's what was meant
Ensure MRs always use branch refs for comparison
Fix issue comment submit button disabled on GFM paste
Lock seed-fu at the correct version in Gemfile.lock
Improve indexes on merge_request_diffs
...
Diffstat (limited to 'lib')
61 files changed, 772 insertions, 264 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index cdef1b546a9..0791a110c39 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -81,9 +81,9 @@ module API service_args = [user_project, current_user, protected_branch_params] protected_branch = if protected_branch - ::ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + ::ProtectedBranches::LegacyApiUpdateService.new(*service_args).execute(protected_branch) else - ::ProtectedBranches::ApiCreateService.new(*service_args).execute + ::ProtectedBranches::LegacyApiCreateService.new(*service_args).execute end if protected_branch.valid? diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 16ae99b5c6c..ce332fe85d2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -80,16 +80,21 @@ module API expose :group_access, as: :group_access_level end - class BasicProjectDetails < Grape::Entity - expose :id, :description, :default_branch, :tag_list - expose :ssh_url_to_repo, :http_url_to_repo, :web_url + class ProjectIdentity < Grape::Entity + expose :id, :description expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :created_at + end + + class BasicProjectDetails < ProjectIdentity + expose :default_branch, :tag_list + expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :avatar_url do |project, options| project.avatar_url(only_path: false) end expose :star_count, :forks_count - expose :created_at, :last_activity_at + expose :last_activity_at end class Project < BasicProjectDetails @@ -242,7 +247,11 @@ module API end expose :merged do |repo_branch, options| - options[:project].repository.merged_to_root_ref?(repo_branch, options[:merged_branch_names]) + if options[:merged_branch_names] + options[:merged_branch_names].include?(repo_branch.name) + else + options[:project].repository.merged_to_root_ref?(repo_branch) + end end expose :protected do |repo_branch, options| @@ -763,7 +772,10 @@ module API expose(:default_project_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_project_visibility) } expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) } expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) } - expose :password_authentication_enabled, as: :signin_enabled + + # support legacy names, can be removed in v5 + expose :password_authentication_enabled_for_web, as: :password_authentication_enabled + expose :password_authentication_enabled_for_web, as: :signin_enabled end class Release < Grape::Entity @@ -820,17 +832,24 @@ module API expose :id, :sha, :ref, :status end - class Job < Grape::Entity + class JobBasic < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at expose :duration expose :user, with: User - expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :commit, with: Commit - expose :runner, with: Runner expose :pipeline, with: PipelineBasic end + class Job < JobBasic + expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } + expose :runner, with: Runner + end + + class JobBasicWithProject < JobBasic + expose :project, with: ProjectIdentity + end + class Trigger < Grape::Entity expose :id expose :token, :description diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8e37ff7f7ce..9ba15893f55 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -56,6 +56,10 @@ module API initial_current_user != current_user end + def user_namespace + @user_namespace ||= find_namespace!(params[:id]) + end + def user_group @group ||= find_group!(params[:id]) end @@ -118,6 +122,24 @@ module API end end + def find_namespace(id) + if id.to_s =~ /^\d+$/ + Namespace.find_by(id: id) + else + Namespace.find_by_full_path(id) + end + end + + def find_namespace!(id) + namespace = find_namespace(id) + + if can?(current_user, :read_namespace, namespace) + namespace + else + not_found!('Namespace') + end + end + def find_project_label(id) label = available_labels.find_by_id(id) || available_labels.find_by_title(id) label || not_found!('Label') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 520bf65c3b3..eff1c5b70ea 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -2,8 +2,8 @@ module API module Helpers module InternalHelpers SSH_GITALY_FEATURES = { - 'git-receive-pack' => :ssh_receive_pack, - 'git-upload-pack' => :ssh_upload_pack + 'git-receive-pack' => [:ssh_receive_pack, Gitlab::GitalyClient::MigrationStatus::OPT_IN], + 'git-upload-pack' => [:ssh_upload_pack, Gitlab::GitalyClient::MigrationStatus::OPT_OUT] }.freeze attr_reader :redirected_path @@ -102,8 +102,8 @@ module API # Return the Gitaly Address if it is enabled def gitaly_payload(action) - feature = SSH_GITALY_FEATURES[action] - return unless feature && Gitlab::GitalyClient.feature_enabled?(feature) + feature, status = SSH_GITALY_FEATURES[action] + return unless feature && Gitlab::GitalyClient.feature_enabled?(feature, status: status) { repository: repository.gitaly_repository, diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 74dfd9f96de..e60e00d7956 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -255,7 +255,9 @@ module API authorize!(:destroy_issue, issue) - destroy_conditionally!(issue) + destroy_conditionally!(issue) do |issue| + Issuable::DestroyService.new(user_project, current_user).execute(issue) + end end desc 'List merge requests closing issue' do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 726f09e3669..d34886fca2e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -21,7 +21,7 @@ module API return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels, :timelogs) + .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) end params :merge_requests_params do @@ -167,7 +167,9 @@ module API authorize!(:destroy_merge_request, merge_request) - destroy_conditionally!(merge_request) + destroy_conditionally!(merge_request) do |merge_request| + Issuable::DestroyService.new(user_project, current_user).execute(merge_request) + end end params do diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index f1eaff6b0eb..32b77aedba8 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -19,6 +19,16 @@ module API present paginate(namespaces), with: Entities::Namespace, current_user: current_user end + + desc 'Get a namespace by ID' do + success Entities::Namespace + end + params do + requires :id, type: String, desc: "Namespace's ID or path" + end + get ':id' do + present user_namespace, with: Entities::Namespace, current_user: current_user + end end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index ceaaeca4046..3588dc85c9e 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -18,6 +18,10 @@ module API end params do requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return notes ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return notes sorted in `asc` or `desc` order.' use :pagination end get ":id/#{noteables_str}/:noteable_id/notes" do @@ -29,11 +33,12 @@ module API # at the DB query level (which we cannot in that case), the current # page can have less elements than :per_page even if # there's more than one page. + raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort]) notes = # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(noteable.notes.with_metadata) + paginate(raw_notes) .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note else diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 15fcb9e8e27..b5021e8a712 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -40,10 +40,10 @@ module API params do requires :name, type: String, desc: 'The name of the protected branch' optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER, - values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to push (defaults: `40`, master access level)' optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER, - values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to merge (defaults: `40`, master access level)' end post ':id/protected_branches' do diff --git a/lib/api/runners.rb b/lib/api/runners.rb index e816fcdd928..996457c5dfe 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -84,6 +84,23 @@ module API destroy_conditionally!(runner) end + + desc 'List jobs running on a runner' do + success Entities::JobBasicWithProject + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES + use :pagination + end + get ':id/jobs' do + runner = get_runner(params[:id]) + authenticate_list_runners_jobs!(runner) + + jobs = RunnerJobsFinder.new(runner, params).execute + + present paginate(jobs), with: Entities::JobBasicWithProject + end end params do @@ -192,6 +209,12 @@ module API forbidden!("No access granted") unless user_can_access_runner?(runner) end + def authenticate_list_runners_jobs!(runner) + return if current_user.admin? + + forbidden!("No access granted") unless user_can_access_runner?(runner) + end + def user_can_access_runner?(runner) current_user.ci_authorized_runners.exists?(runner.id) end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 851b226e9e5..cee4d309816 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -44,9 +44,11 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - mutually_exclusive :password_authentication_enabled, :signin_enabled + optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled + optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)' optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -121,6 +123,9 @@ module API end optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", @@ -135,8 +140,11 @@ module API put "application/settings" do attrs = declared_params(include_missing: false) + # support legacy names, can be removed in v5 if attrs.has_key?(:signin_enabled) - attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:signin_enabled) + elsif attrs.has_key?(:password_authentication_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end if current_settings.update_attributes(attrs) diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index afdd7b83998..c17b6f45ed8 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -172,8 +172,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :password_authentication_enabled - expose :password_authentication_enabled, as: :signin_enabled + expose :password_authentication_enabled_for_web, as: :password_authentication_enabled + expose :password_authentication_enabled_for_web, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb index 202011cfcbe..9b4ab7630fb 100644 --- a/lib/api/v3/settings.rb +++ b/lib/api/v3/settings.rb @@ -44,8 +44,8 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do @@ -131,7 +131,9 @@ module API attrs = declared_params(include_missing: false) if attrs.has_key?(:signin_enabled) - attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:signin_enabled) + elsif attrs.has_key?(:password_authentication_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end if current_settings.update_attributes(attrs) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 9fef386de16..8975395aff1 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -213,7 +213,8 @@ module Banzai end def object_link_text(object, matches) - text = object.reference_link_text(context[:project]) + parent = context[:project] || context[:group] + text = object.reference_link_text(parent) extras = object_link_text_extras(object, matches) text += " (#{extras.join(", ")})" if extras.any? diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb new file mode 100644 index 00000000000..b545b947a2c --- /dev/null +++ b/lib/banzai/filter/mermaid_filter.rb @@ -0,0 +1,20 @@ +module Banzai + module Filter + class MermaidFilter < HTML::Pipeline::Filter + def call + doc.css('pre[lang="mermaid"]').add_class('mermaid') + doc.css('pre[lang="mermaid"]').add_class('js-render-mermaid') + + # The `<code></code>` blocks are added in the lib/banzai/filter/syntax_highlight_filter.rb + # We want to keep context and consistency, so we the blocks are added for all filters. + # Details: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15107/diffs?diff_id=7962900#note_45495859 + doc.css('pre[lang="mermaid"]').each do |pre| + document = pre.at('code') + document.replace(document.content) + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 7da565043d1..a79a0154846 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,23 +14,26 @@ module Banzai end def highlight_node(node) - language = node.attr('lang') code = node.text - css_classes = "code highlight" - lexer = lexer_for(language) - lang = lexer.tag - - begin - code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: lang) - - css_classes << " js-syntax-highlight #{lang}" - rescue - lang = nil - # Gracefully handle syntax highlighter bugs/errors to ensure - # users can still access an issue/comment/etc. + css_classes = 'code highlight js-syntax-highlight' + language = node.attr('lang') + + if use_rouge?(language) + lexer = lexer_for(language) + language = lexer.tag + + begin + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: language) + css_classes << " #{language}" + rescue + # Gracefully handle syntax highlighter bugs/errors to ensure + # users can still access an issue/comment/etc. + + language = nil + end end - highlighted = %(<pre class="#{css_classes}" lang="#{lang}" v-pre="true"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" lang="#{language}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) @@ -51,6 +54,10 @@ module Banzai # Replace the parent `pre` element with the entire highlighted block node.parent.replace(highlighted) end + + def use_rouge?(language) + %w(math mermaid plantuml).exclude?(language) + end end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 3208abfc538..55874ad50a3 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -14,6 +14,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::MathFilter, + Filter::MermaidFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index cbbc51db99e..65d7fd3ec70 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -34,7 +34,7 @@ module Gitlab rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + return result if result.success? || authenticate_using_internal_or_ldap_password? # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts @@ -45,6 +45,10 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless authenticate_using_internal_or_ldap_password? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -52,10 +56,8 @@ module Gitlab # LDAP users are only authenticated via LDAP if user.nil? || user.ldap_user? # Second chance - try LDAP authentication - return unless Gitlab::LDAP::Config.enabled? - Gitlab::LDAP::Authentication.login(login, password) - else + elsif current_application_settings.password_authentication_enabled_for_git? user if user.active? && user.valid_password?(password) end end @@ -84,6 +86,10 @@ module Gitlab private + def authenticate_using_internal_or_ldap_password? + current_application_settings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + end + def service_request_check(login, password, project) matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) @@ -128,7 +134,7 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) if token && valid_scoped_token?(token, available_scopes) - Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scope(token.scopes)) + Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end @@ -140,10 +146,15 @@ module Gitlab AccessTokenValidationService.new(token).include_any_scope?(scopes) end - def abilities_for_scope(scopes) - scopes.map do |scope| - self.public_send(:"#{scope}_scope_authentication_abilities") # rubocop:disable GitlabSecurity/PublicSend - end.flatten.uniq + def abilities_for_scopes(scopes) + abilities_by_scope = { + api: full_authentication_abilities, + read_registry: [:read_container_image] + } + + scopes.flat_map do |scope| + abilities_by_scope.fetch(scope.to_sym, []) + end.uniq end def lfs_token_check(login, password, project) @@ -222,16 +233,6 @@ module Gitlab :admin_container_image ] end - alias_method :api_scope_authentication_abilities, :full_authentication_abilities - - def read_registry_scope_authentication_abilities - [:read_container_image] - end - - # The currently used auth method doesn't allow any actions for this scope - def read_user_scope_authentication_abilities - [] - end def available_scopes(current_user = nil) scopes = API_SCOPES + registry_scopes diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml new file mode 100644 index 00000000000..8242821cedc --- /dev/null +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -0,0 +1,52 @@ +# For background migrations we define a custom set of rules to make it less +# difficult to review these migrations. To reduce the complexity of these +# migrations some rules may be stricter than the defaults set in the root +# .rubocop.yml file. +--- +inherit_from: ../../../.rubocop.yml + +Metrics/AbcSize: + Enabled: true + Max: 30 + Details: > + Code that involves a lot of branches can be very hard to wrap your head + around. + +Metrics/PerceivedComplexity: + Enabled: true + +Metrics/LineLength: + Enabled: true + Details: > + Long lines are very hard to read and make it more difficult to review + changes. + +Metrics/MethodLength: + Enabled: true + Max: 30 + Details: > + Long methods can be very hard to review. Consider splitting this method up + into separate methods. + +Metrics/ClassLength: + Enabled: true + Details: > + Long classes can be very hard to review. Consider splitting this class up + into multiple classes. + +Metrics/BlockLength: + Enabled: true + Details: > + Long blocks can be hard to read. Consider splitting the code into separate + methods. + +Style/Documentation: + Enabled: true + Details: > + Adding documentation makes it easier to figure out what a migration is + supposed to do. + +Style/FrozenStringLiteralComment: + Enabled: true + Details: >- + This removes the need for calling "freeze", reducing noise in the code. diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 67a39d28944..03b17b319fa 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class CreateForkNetworkMembershipsRange diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb index e94719db72e..c2bf42f846d 100644 --- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys class GpgKey < ActiveRecord::Base self.table_name = 'gpg_keys' diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb index b1411be3016..a1af045a71f 100644 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class DeleteConflictingRedirectRoutesRange diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 380802258f5..fd5cbf76e47 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -1,3 +1,9 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class DeserializeMergeRequestDiffsAndCommits diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index 91540127ea9..0a8a4313cd5 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateBuildStageIdReference diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 432f7c3e706..84ac00f1a5c 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration # Class that migrates events for the new push event payloads setup. All diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index b1ff0900709..0e5c7f092f2 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateStageStatus diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb index 0881244ed49..7f243073fd0 100644 --- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateSystemUploadsToNewFolder diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb index 07cec96bcc3..a4ef51fd0e8 100644 --- a/lib/gitlab/background_migration/move_personal_snippet_files.rb +++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MovePersonalSnippetFiles diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb index bc53e6d7f94..85749366bfd 100644 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -1,3 +1,10 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/ClassLength +# rubocop:disable Metrics/BlockLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class NormalizeLdapExternUidsRange diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb index 2ef3a207dd3..a976cb4c243 100644 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -1,25 +1,55 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration + # This background migration is going to create all `fork_networks` and + # the `fork_network_members` for the roots of fork networks based on the + # existing `forked_project_links`. + # + # When the source of a fork is deleted, we will create the fork with the + # target project as the root. This way, when there are forks of the target + # project, they will be joined into the same fork network. + # + # When the `fork_networks` and memberships for the root projects are created + # the `CreateForkNetworkMembershipsRange` migration is scheduled. This + # migration will create the memberships for all remaining forks-of-forks class PopulateForkNetworksRange def perform(start_id, end_id) - log("Creating fork networks for forked project links: #{start_id} - #{end_id}") + create_fork_networks_for_existing_projects(start_id, end_id) + create_fork_networks_for_missing_projects(start_id, end_id) + create_fork_networks_memberships_for_root_projects(start_id, end_id) + + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength + BackgroundMigrationWorker.perform_in( + delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] + ) + end + def create_fork_networks_for_existing_projects(start_id, end_id) + log("Creating fork networks: #{start_id} - #{end_id}") ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS INSERT INTO fork_networks (root_project_id) SELECT DISTINCT forked_project_links.forked_from_project_id FROM forked_project_links + -- Exclude the forks that are not the first level fork of a project WHERE NOT EXISTS ( SELECT true FROM forked_project_links inner_links WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id ) + + /* Exclude the ones that are already created, in case the fork network + was already created for another fork of the project. + */ AND NOT EXISTS ( SELECT true FROM fork_networks WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id ) + + -- Only create a fork network for a root project that still exists AND EXISTS ( SELECT true FROM projects @@ -27,7 +57,45 @@ module Gitlab ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} INSERT_NETWORKS + end + + def create_fork_networks_for_missing_projects(start_id, end_id) + log("Creating fork networks with missing root: #{start_id} - #{end_id}") + ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS + INSERT INTO fork_networks (root_project_id) + SELECT DISTINCT forked_project_links.forked_to_project_id + + FROM forked_project_links + + -- Exclude forks that are not the root forks + WHERE NOT EXISTS ( + SELECT true + FROM forked_project_links inner_links + WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id + ) + + /* Exclude the ones that are already created, in case this migration is + re-run + */ + AND NOT EXISTS ( + SELECT true + FROM fork_networks + WHERE forked_project_links.forked_to_project_id = fork_networks.root_project_id + ) + + /* Exclude projects for which the project still exists, those are + Processed in the previous step of this migration + */ + AND NOT EXISTS ( + SELECT true + FROM projects + WHERE projects.id = forked_project_links.forked_from_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_NETWORKS + end + def create_fork_networks_memberships_for_root_projects(start_id, end_id) log("Creating memberships for root projects: #{start_id} - #{end_id}") ActiveRecord::Base.connection.execute <<~INSERT_ROOT @@ -36,8 +104,12 @@ module Gitlab FROM fork_networks + /* Joining both on forked_from- and forked_to- so we could create the + memberships for forks for which the source was deleted + */ INNER JOIN forked_project_links ON forked_project_links.forked_from_project_id = fork_networks.root_project_id + OR forked_project_links.forked_to_project_id = fork_networks.root_project_id WHERE NOT EXISTS ( SELECT true @@ -46,9 +118,6 @@ module Gitlab ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} INSERT_ROOT - - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY - BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]) end def log(message) diff --git a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb index 7e109e96e73..dcac355e1b0 100644 --- a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb +++ b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class PopulateMergeRequestsLatestMergeRequestDiffId diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index ca2742f2aae..086203b9ccc 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -3,7 +3,6 @@ module Gitlab class PlanEventFetcher < BaseEventFetcher def initialize(*args) @projections = [mr_diff_table[:id], - mr_diff_table[:st_commits], issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) @@ -41,12 +40,7 @@ module Gitlab def first_time_reference_commit(event) return nil unless event && merge_request_diff_commits - commits = - if event['st_commits'].present? - YAML.load(event['st_commits']) - else - merge_request_diff_commits[event['id'].to_i] - end + commits = merge_request_diff_commits[event['id'].to_i] return nil if commits.blank? diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index efc2e46d289..4a9d3e52fae 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -31,16 +31,22 @@ module Gitlab def check ensure_patches_dir - generate_patch(ce_branch, ce_patch_full_path) + add_remote('canonical-ce', "#{DEFAULT_CE_PROJECT_URL}.git") + generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, remote: 'canonical-ce') ensure_ee_repo Dir.chdir(ee_repo_dir) do step("In the #{ee_repo_dir} directory") + add_remote('canonical-ee', EE_REPO_URL) + status = catch(:halt_check) do ce_branch_compat_check! delete_ee_branches_locally! ee_branch_presence_check! + + step("Checking out #{ee_branch_found}", %W[git checkout -b #{ee_branch_found} canonical-ee/#{ee_branch_found}]) + generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, remote: 'canonical-ee') ee_branch_compat_check! end @@ -56,6 +62,13 @@ module Gitlab private + def add_remote(name, url) + step( + "Adding the #{name} remote (#{url})", + %W[git remote add #{name} #{url}] + ) + end + def ensure_ee_repo if Dir.exist?(ee_repo_dir) step("#{ee_repo_dir} already exists") @@ -71,14 +84,14 @@ module Gitlab FileUtils.mkdir_p(patches_dir) end - def generate_patch(branch, patch_path) + def generate_patch(branch:, patch_path:, remote:) FileUtils.rm(patch_path, force: true) - find_merge_base_with_master(branch: branch) + find_merge_base_with_master(branch: branch, master_remote: remote) step( - "Generating the patch against origin/master in #{patch_path}", - %w[git diff --binary origin/master...HEAD] + "Generating the patch against #{remote}/master in #{patch_path}", + %W[git diff --binary #{remote}/master...origin/#{branch}] ) do |output, status| throw(:halt_check, :ko) unless status.zero? @@ -96,14 +109,14 @@ module Gitlab end def ee_branch_presence_check! - _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch origin #{ee_branch_prefix}]) + _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch canonical-ee #{ee_branch_prefix}]) if status.zero? @ee_branch_found = ee_branch_prefix return end - _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}]) + _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch canonical-ee #{ee_branch_suffix}]) if status.zero? @ee_branch_found = ee_branch_suffix @@ -116,10 +129,6 @@ module Gitlab end def ee_branch_compat_check! - step("Checking out origin/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} FETCH_HEAD]) - - generate_patch(ee_branch_found, ee_patch_full_path) - unless check_patch(ee_patch_full_path).zero? puts puts ee_branch_doesnt_apply_cleanly_msg @@ -133,8 +142,7 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) - step("Resetting to latest master", %w[git reset --hard origin/master]) - step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo_url} #{ce_branch}]) + step("Resetting to latest master", %w[git reset --hard canonical-ee/master]) step( "Checking if #{patch_path} applies cleanly to EE/master", # Don't use --check here because it can result in a 0-exit status even @@ -171,10 +179,10 @@ module Gitlab command(%W[git branch --delete --force #{ee_branch_suffix}]) end - def merge_base_found? + def merge_base_found?(master_remote:, branch:) step( - "Finding merge base with master", - %w[git merge-base origin/master HEAD] + "Finding merge base with #{master_remote}/master", + %W[git merge-base #{master_remote}/master origin/#{branch}] ) do |output, status| if status.zero? puts "Merge base was found: #{output}" @@ -183,7 +191,7 @@ module Gitlab end end - def find_merge_base_with_master(branch:) + def find_merge_base_with_master(branch:, master_remote:) # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 @@ -192,19 +200,19 @@ module Gitlab depth += Math.exp(factor).to_i # Repository is initially cloned with a depth of 20 so we need to fetch # deeper in the case the branch has more than 20 commits on top of master - fetch(branch: branch, depth: depth) - fetch(branch: 'master', depth: depth, remote: DEFAULT_CE_PROJECT_URL) + fetch(branch: branch, depth: depth, remote: 'origin') + fetch(branch: 'master', depth: depth, remote: master_remote) - merge_base_found? + merge_base_found?(master_remote: master_remote, branch: branch) end - raise "\n#{branch} is too far behind master, please rebase it!\n" unless success + raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success end def fetch(branch:, depth:, remote: 'origin') step( "Fetching deeper...", - %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/origin/#{branch}] + %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/#{remote}/#{branch}] ) do |output, status| raise "Fetch failed: #{output}" unless status.zero? end @@ -304,8 +312,8 @@ module Gitlab 1. Create a new branch from master and cherry-pick your CE commits # In the EE repo - $ git fetch origin - $ git checkout -b #{ee_branch_prefix} origin/master + $ git fetch #{EE_REPO_URL} master + $ git checkout -b #{ee_branch_prefix} FETCH_HEAD $ git fetch #{ce_repo_url} #{ce_branch} $ git cherry-pick SHA # Repeat for all the commits you want to pick @@ -314,10 +322,9 @@ module Gitlab 2. Apply your branch's patch to EE # In the EE repo - $ git fetch origin master - $ git checkout -b #{ee_branch_prefix} origin/master - $ wget #{patch_url} - $ git apply --3way #{ce_patch_name} + $ git fetch #{EE_REPO_URL} master + $ git checkout -b #{ee_branch_prefix} FETCH_HEAD + $ wget #{patch_url} && git apply --3way #{ce_patch_name} At this point you might have conflicts such as: diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 99dfee3dd9b..582028493e9 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -17,6 +17,10 @@ module Gitlab return nil unless message.respond_to?(:force_encoding) return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + if message.respond_to?(:frozen?) && message.frozen? + message = message.dup + end + message.force_encoding("UTF-8") return message if message.valid_encoding? diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3cb9b254e6e..d399636bb28 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1046,9 +1046,15 @@ module Gitlab end def with_repo_tmp_commit(start_repository, start_branch_name, sha) + source_ref = start_branch_name + + unless Gitlab::Git.branch_ref?(source_ref) + source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" + end + tmp_ref = fetch_ref( start_repository, - source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + source_ref: source_ref, target_ref: "refs/tmp/#{SecureRandom.hex}" ) @@ -1058,12 +1064,11 @@ module Gitlab end def fetch_source_branch!(source_repository, source_branch, local_ref) - with_repo_branch_commit(source_repository, source_branch) do |commit| - if commit - write_ref(local_ref, commit.sha) - true + Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled| + if is_enabled + gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) else - false + rugged_fetch_source_branch(source_repository, source_branch, local_ref) end end end @@ -1151,10 +1156,12 @@ module Gitlab @has_visible_content = has_local_branches? end - def fetch(remote = 'origin') - args = %W(#{Gitlab.config.git.bin_path} fetch #{remote}) - - popen(args, @path).last.zero? + # Like all public `Gitlab::Git::Repository` methods, this method is part + # of `Repository`'s interface through `method_missing`. + # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and + # takes some extra attributes, so we qualify this method name to prevent confusion. + def fetch_remote_without_shell(remote = 'origin') + run_git(['fetch', remote]).last.zero? end def blob_at(sha, path) @@ -1216,6 +1223,17 @@ module Gitlab private + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) + with_repo_branch_commit(source_repository, source_branch) do |commit| + if commit + write_ref(local_ref, commit.sha) + true + else + false + end + end + end + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 @@ -1233,11 +1251,21 @@ module Gitlab sort_branches(branches, sort_by) end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 def git_merged_branch_names(branch_names = []) - lines = run_git(['branch', '--merged', root_ref] + branch_names) - .first.lines + root_sha = find_branch(root_ref).target + + git_arguments = + %W[branch --merged #{root_sha} + --format=%(refname:short)\ %(objectname)] + branch_names - lines.map(&:strip) + lines = run_git(git_arguments).first.lines + + lines.each_with_object([]) do |line, branches| + name, sha = line.strip.split(' ', 2) + + branches << name if sha != root_sha + end end def log_using_shell?(options) diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index 4500482d68f..392bef69e80 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -1,38 +1,47 @@ module Gitlab module Git module RepositoryMirroring - IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze - IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze - MIRROR_REMOTE = 'mirror'.freeze + REFMAPS = { + # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror` + all_refs: '+refs/*:refs/*', + heads: '+refs/heads/*:refs/heads/*', + tags: '+refs/tags/*:refs/tags/*' + }.freeze RemoteError = Class.new(StandardError) - def set_remote_as_mirror(remote_name) - # This is used to define repository as equivalent as "git clone --mirror" - rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*' - rugged.config["remote.#{remote_name}.mirror"] = true - rugged.config["remote.#{remote_name}.prune"] = true - end - - def set_import_remote_as_mirror(remote_name) - # Add first fetch with Rugged so it does not create its own. - rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS - - add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) + def set_remote_as_mirror(remote_name, refmap: :all_refs) + set_remote_refmap(remote_name, refmap) rugged.config["remote.#{remote_name}.mirror"] = true rugged.config["remote.#{remote_name}.prune"] = true end - def add_remote_fetch_config(remote_name, refspec) - run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + def set_remote_refmap(remote_name, refmap) + Array(refmap).each_with_index do |refspec, i| + refspec = REFMAPS[refspec] || refspec + + # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it. + # To make sure we start from scratch, we set the first using rugged, and use `git` for any others + if i == 0 + rugged.config["remote.#{remote_name}.fetch"] = refspec + else + run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + end + end end - def fetch_mirror(url) - add_remote(MIRROR_REMOTE, url) - set_remote_as_mirror(MIRROR_REMOTE) - fetch(MIRROR_REMOTE) - remove_remote(MIRROR_REMOTE) + # Like all_refs public `Gitlab::Git::Repository` methods, this method is part + # of `Repository`'s interface through `method_missing`. + # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and + # takes some extra attributes, so we qualify this method name to prevent confusion. + def fetch_as_mirror_without_shell(url) + remote_name = "tmp-#{SecureRandom.hex}" + add_remote(remote_name, url) + set_remote_as_mirror(remote_name) + fetch_remote_without_shell(remote_name) + ensure + remove_remote(remote_name) if remote_name end def remote_tags(remote) diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index e6b61417de1..e573cd0e143 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -8,7 +8,12 @@ module Gitlab end def self.from_gitaly(gitaly_user) - new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new( + gitaly_user.gl_username, + Gitlab::EncodingHelper.encode!(gitaly_user.name), + Gitlab::EncodingHelper.encode!(gitaly_user.email), + gitaly_user.gl_id + ) end def initialize(username, name, email, gl_id) @@ -23,7 +28,7 @@ module Gitlab end def to_gitaly - Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email) + Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b) end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 0b35a787e07..f27cd800bdd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -31,14 +31,38 @@ module Gitlab CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new - private_constant :MUTEX + METRICS_MUTEX = Mutex.new + private_constant :MUTEX, :METRICS_MUTEX class << self - attr_accessor :query_time, :migrate_histogram + attr_accessor :query_time end self.query_time = 0 - self.migrate_histogram = Gitlab::Metrics.histogram(:gitaly_migrate_call_duration, "Gitaly migration call execution timings") + + def self.migrate_histogram + @migrate_histogram ||= + METRICS_MUTEX.synchronize do + # If a thread was blocked on the mutex, the value was set already + return @migrate_histogram if @migrate_histogram + + Gitlab::Metrics.histogram(:gitaly_migrate_call_duration_seconds, + "Gitaly migration call execution timings", + gitaly_enabled: nil, feature: nil) + end + end + + def self.gitaly_call_histogram + @gitaly_call_histogram ||= + METRICS_MUTEX.synchronize do + # If a thread was blocked on the mutex, the value was set already + return @gitaly_call_histogram if @gitaly_call_histogram + + Gitlab::Metrics.histogram(:gitaly_controller_action_duration_seconds, + "Gitaly endpoint histogram by controller and action combination", + Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil)) + end + end def self.stub(name, storage) MUTEX.synchronize do @@ -75,6 +99,10 @@ module Gitlab address end + def self.address_metadata(storage) + Base64.strict_encode64(JSON.dump({ storage => { 'address' => address(storage), 'token' => token(storage) } })) + end + # All Gitaly RPC call sites should use GitalyClient.call. This method # makes sure that per-request authentication headers are set. # @@ -89,18 +117,30 @@ module Gitlab # kwargs.merge(deadline: Time.now + 10) # end # - def self.call(storage, service, rpc, request) - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil) + start = Gitlab::Metrics::System.monotonic_time enforce_gitaly_request_limits(:call) - kwargs = request_kwargs(storage) + kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? + stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend ensure - self.query_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + duration = Gitlab::Metrics::System.monotonic_time - start + + # Keep track, seperately, for the performance bar + self.query_time += duration + gitaly_call_histogram.observe( + current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), + duration) + end + + def self.current_transaction_labels + Gitlab::Metrics::Transaction.current&.labels || {} end + private_class_method :current_transaction_labels - def self.request_kwargs(storage) + def self.request_kwargs(storage, timeout, remote_storage: nil) encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { 'authorization' => "Bearer #{encoded_token}", @@ -110,8 +150,24 @@ module Gitlab feature_stack = Thread.current[:gitaly_feature_stack] feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature + metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage - { metadata: metadata } + result = { metadata: metadata } + + # nil timeout indicates that we should use the default + timeout = default_timeout if timeout.nil? + + return result unless timeout > 0 + + # Do not use `Time.now` for deadline calculation, since it + # will be affected by Timecop in some tests, but grpc's c-core + # uses system time instead of timecop's time, so tests will fail + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will + # circumvent timecop + deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + result[:deadline] = deadline + + result end def self.token(storage) @@ -172,10 +228,10 @@ module Gitlab feature_stack = Thread.current[:gitaly_feature_stack] ||= [] feature_stack.unshift(feature) begin - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + start = Gitlab::Metrics::System.monotonic_time yield is_enabled ensure - total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + total_time = Gitlab::Metrics::System.monotonic_time - start migrate_histogram.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) feature_stack.shift Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? @@ -284,6 +340,26 @@ module Gitlab Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) end + # The default timeout on all Gitaly calls + def self.default_timeout + return 0 if Sidekiq.server? + + timeout(:gitaly_timeout_default) + end + + def self.fast_timeout + timeout(:gitaly_timeout_fast) + end + + def self.medium_timeout + timeout(:gitaly_timeout_medium) + end + + def self.timeout(timeout_name) + Gitlab::CurrentSettings.current_application_settings[timeout_name] + end + private_class_method :timeout + # Count a stack. Used for n+1 detection def self.count_stack return unless RequestStore.active? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index da5505cb2fe..34807d280e5 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -16,7 +16,7 @@ module Gitlab revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request) + response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map { |d| EncodingHelper.encode!(d.dup) } end @@ -29,7 +29,7 @@ module Gitlab child_id: child_id ) - GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value + GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value end def diff(from, to, options = {}) @@ -77,7 +77,7 @@ module Gitlab limit: limit.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) + response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) entry = nil data = '' @@ -102,7 +102,7 @@ module Gitlab path: path.present? ? GitalyClient.encode(path) : '.' ) - response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) + response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) response.flat_map do |message| message.entries.map do |gitaly_tree_entry| @@ -129,7 +129,7 @@ module Gitlab request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? request.path = options[:path] if options[:path].present? - GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count + GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end def last_commit_for_path(revision, path) @@ -139,7 +139,7 @@ module Gitlab path: GitalyClient.encode(path.to_s) ) - gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit + gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit return unless gitaly_commit Gitlab::Git::Commit.new(@repository, gitaly_commit) @@ -152,7 +152,7 @@ module Gitlab to: to ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request) + response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -165,7 +165,7 @@ module Gitlab ) request.order = opts[:order].upcase if opts[:order].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -179,7 +179,7 @@ module Gitlab offset: offset.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request) + response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -197,7 +197,7 @@ module Gitlab path: GitalyClient.encode(path) ) - response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request) + response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce("") { |memo, msg| memo << msg.data } end @@ -207,7 +207,7 @@ module Gitlab revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) response.commit end @@ -217,7 +217,7 @@ module Gitlab repository: @gitaly_repo, revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout) response.sum(&:data) end @@ -227,7 +227,7 @@ module Gitlab repository: @gitaly_repo, revision: GitalyClient.encode(revision) ) - GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) + GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end def find_commits(options) @@ -245,7 +245,7 @@ module Gitlab request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -259,7 +259,7 @@ module Gitlab request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) GitalyClient::DiffStitcher.new(response) end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 31b04bc2650..066e4e183c0 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -46,7 +46,8 @@ module Gitlab commit_id: commit_id, prefix: ref_prefix ) - encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup) + response = GitalyClient.call(@storage, :ref_service, :find_ref_name, request, timeout: GitalyClient.medium_timeout) + encode!(response.name.dup) end def count_tag_names diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index cef692d3c2a..b9e606592d7 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -10,7 +10,9 @@ module Gitlab def exists? request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repository_exists, request).exists + response = GitalyClient.call(@storage, :repository_service, :repository_exists, request, timeout: GitalyClient.fast_timeout) + + response.exists end def garbage_collect(create_bitmap) @@ -30,7 +32,8 @@ module Gitlab def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repository_size, request).size + response = GitalyClient.call(@storage, :repository_service, :repository_size, request) + response.size end def apply_gitattributes(revision) @@ -61,10 +64,29 @@ module Gitlab def has_local_branches? request = Gitaly::HasLocalBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request) + response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request, timeout: GitalyClient.fast_timeout) response.value end + + def fetch_source_branch(source_repository, source_branch, local_ref) + request = Gitaly::FetchSourceBranchRequest.new( + repository: @gitaly_repo, + source_repository: source_repository.gitaly_repository, + source_branch: source_branch.b, + target_ref: local_ref.b + ) + + response = GitalyClient.call( + @storage, + :repository_service, + :fetch_source_branch, + request, + remote_storage: source_repository.storage + ) + + response.result + end end end end diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index d2ae4c1255e..65b5e30c70f 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -1,5 +1,9 @@ module Gitlab module GithubImport + def self.refmap + [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head'] + end + def self.new_client_for(project, token: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 0b67fc8db73..9cf2e7fd871 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -45,27 +45,14 @@ module Gitlab def import_repository project.ensure_repository - configure_repository_remote - - project.repository.fetch_remote('github', forced: true) + refmap = Gitlab::GithubImport.refmap + project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true, remote_name: 'github') true rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e fail_import("Failed to import the repository: #{e.message}") end - def configure_repository_remote - return if project.repository.remote_exists?('github') - - project.repository.add_remote('github', project.import_url) - project.repository.set_import_remote_as_mirror('github') - - project.repository.add_remote_fetch_config( - 'github', - '+refs/pull/*/head:refs/merge-requests/*/head' - ) - end - def import_wiki_repository wiki_path = "#{project.disk_path}.wiki" wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 50ee879129c..2066005dddc 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.0'.freeze + VERSION = '0.2.1'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 263599831bf..f2b193c79cb 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -133,8 +133,6 @@ methods: - :type services: - :type - merge_request_diff: - - :utf8_st_diffs merge_request_diff_files: - :utf8_diff merge_requests: diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 639f4f0c3f0..c518943be59 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -60,6 +60,8 @@ module Gitlab end end + @project.merge_requests.set_latest_merge_request_diff_ids! + @saved end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 2b34ceb5831..d7d1b05e8b9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -58,7 +58,6 @@ module Gitlab def setup_models case @relation_name - when :merge_request_diff then setup_st_diff_commits when :merge_request_diff_files then setup_diff when :notes then setup_note when :project_label, :project_labels then setup_label @@ -208,13 +207,6 @@ module Gitlab relation_class: relation_class) end - def setup_st_diff_commits - @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') - - HashUtil.deep_symbolize_array!(@relation_hash['st_diffs']) - HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) - end - def setup_diff @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index f9ae5079d7c..627a487d577 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -24,8 +24,7 @@ module Gitlab end def uploads_path - # TODO: decide what to do with uploads. We will use UUIDs here too? - File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) + FileUploader.dynamic_path_segment(@project) end end end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 4d096e5a741..0526ef9eb13 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -3,6 +3,10 @@ module Gitlab class Importer include Gitlab::ShellAdapter + def self.refmap + Gitlab::GithubImport.refmap + end + attr_reader :errors, :project, :repo, :repo_url def initialize(project) diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 90235095306..65d55576ac2 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -6,29 +6,15 @@ module Gitlab BASE_LABELS = { module: nil, method: nil }.freeze attr_reader :real_time, :cpu_time, :call_count, :labels - def self.call_real_duration_histogram - return @call_real_duration_histogram if @call_real_duration_histogram - - MUTEX.synchronize do - @call_real_duration_histogram ||= Gitlab::Metrics.histogram( - :gitlab_method_call_real_duration_seconds, - 'Method calls real duration', - Transaction::BASE_LABELS.merge(BASE_LABELS), - [0.1, 0.2, 0.5, 1, 2, 5, 10] - ) - end - end - - def self.call_cpu_duration_histogram - return @call_cpu_duration_histogram if @call_cpu_duration_histogram + def self.call_duration_histogram + return @call_duration_histogram if @call_duration_histogram MUTEX.synchronize do @call_duration_histogram ||= Gitlab::Metrics.histogram( - :gitlab_method_call_cpu_duration_seconds, - 'Method calls cpu duration', + :gitlab_method_call_duration_seconds, + 'Method calls real duration', Transaction::BASE_LABELS.merge(BASE_LABELS), - [0.1, 0.2, 0.5, 1, 2, 5, 10] - ) + [0.01, 0.05, 0.1, 0.5, 1]) end end @@ -59,8 +45,9 @@ module Gitlab @cpu_time += cpu_time @call_count += 1 - self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) - self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0) + if call_measurement_enabled? && above_threshold? + self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) + end retval end @@ -83,6 +70,10 @@ module Gitlab def above_threshold? real_time >= Metrics.method_call_threshold end + + def call_measurement_enabled? + Feature.get(:prometheus_metrics_method_instrumentation).enabled? + end end end end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 9a91f8bf96a..7e5dfd33502 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -51,7 +51,6 @@ module Gitlab slash-command-logo.png snippets u - unicorn_test unsubscribes uploads users diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index efe8095beea..fef9d3e31d4 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -30,7 +30,7 @@ module Gitlab def initialize(current_user, limit_projects, query) @current_user = current_user @limit_projects = limit_projects || Project.all - @query = Shellwords.shellescape(query) if query.present? + @query = query end def objects(scope, page = nil) diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 7c2d1d8f887..5f0c98cb5a4 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -4,9 +4,15 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_WORD = /(?<=^| )"[^"]+"(?= |$)/ + REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/ class_methods do + def fuzzy_search(query, columns) + matches = columns.map { |col| fuzzy_arel_match(col, query) }.compact.reduce(:or) + + where(matches) + end + def to_pattern(query) if partial_matching?(query) "%#{sanitize_sql_like(query)}%" @@ -19,12 +25,19 @@ module Gitlab query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end - def to_fuzzy_arel(column, query) - words = select_fuzzy_words(query) + def fuzzy_arel_match(column, query) + query = query.squish + return nil unless query.present? - matches = words.map { |word| arel_table[column].matches(to_pattern(word)) } + words = select_fuzzy_words(query) - matches.reduce { |result, match| result.and(match) } + if words.any? + words.map { |word| arel_table[column].matches(to_pattern(word)) }.reduce(:and) + else + # No words of at least 3 chars, but we can search for an exact + # case insensitive match with the query as a whole + arel_table[column].matches(sanitize_sql_like(query)) + end end def select_fuzzy_words(query) @@ -32,7 +45,7 @@ module Gitlab query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } - words = query.split(/\s+/) + words = query.split quoted_words.map! { |quoted_word| quoted_word[1..-2] } diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 112d4939582..2adcc9809b3 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -79,7 +79,7 @@ module Gitlab def features_usage_data_ce { - signup: current_application_settings.signup_enabled?, + signup: current_application_settings.allow_signup?, ldap: Gitlab.config.ldap.enabled, gravatar: current_application_settings.gravatar_enabled?, omniauth: Gitlab.config.omniauth.enabled, diff --git a/lib/milestone_array.rb b/lib/milestone_array.rb new file mode 100644 index 00000000000..4ed8485b36a --- /dev/null +++ b/lib/milestone_array.rb @@ -0,0 +1,40 @@ +module MilestoneArray + class << self + def sort(array, sort_method) + case sort_method + when 'due_date_asc' + sort_asc_nulls_last(array, 'due_date') + when 'due_date_desc' + sort_desc_nulls_last(array, 'due_date') + when 'start_date_asc' + sort_asc_nulls_last(array, 'start_date') + when 'start_date_desc' + sort_desc_nulls_last(array, 'start_date') + when 'name_asc' + sort_asc(array, 'title') + when 'name_desc' + sort_asc(array, 'title').reverse + else + array + end + end + + private + + def sort_asc_nulls_last(array, attribute) + attribute = attribute.to_sym + + array.select(&attribute).sort_by(&attribute) + array.reject(&attribute) + end + + def sort_desc_nulls_last(array, attribute) + attribute = attribute.to_sym + + array.select(&attribute).sort_by(&attribute).reverse + array.reject(&attribute) + end + + def sort_asc(array, attribute) + array.sort_by(&attribute.to_sym) + end + end +end diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb deleted file mode 100644 index 939b23a3421..00000000000 --- a/lib/rouge/lexers/math.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Rouge - module Lexers - class Math < PlainText - title "A passthrough lexer used for LaTeX input" - desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter" - tag 'math' - end - end -end diff --git a/lib/rouge/lexers/plantuml.rb b/lib/rouge/lexers/plantuml.rb deleted file mode 100644 index 63c461764fc..00000000000 --- a/lib/rouge/lexers/plantuml.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Rouge - module Lexers - class Plantuml < PlainText - title "A passthrough lexer used for PlantUML input" - desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter" - tag 'plantuml' - end - end -end diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 99b3168d9eb..2301ec9b228 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 301affc9522..eb0f757aea7 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,11 +1,14 @@ namespace :gitlab do namespace :cleanup do + HASHED_REPOSITORY_NAME = '@hashed'.freeze + desc "GitLab | Cleanup | Clean namespaces" task dirs: :environment do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) + namespaces = Namespace.pluck(:path) + namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| git_base_path = repository_storage['path'] all_dirs = Dir.glob(git_base_path + '/*') @@ -62,7 +65,7 @@ namespace :gitlab do # TODO ignoring hashed repositories for now. But revisit to fully support # possible orphaned hashed repos - next if repo_with_namespace.start_with?('@hashed/') || Project.find_by_full_path(repo_with_namespace) + next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace) new_path = path + move_suffix puts path.inspect + ' -> ' + new_path.inspect diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index f2002d7a426..4d880c05f99 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -78,6 +78,8 @@ namespace :gitlab do config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } + config[:bin_dir] = Gitlab.config.gitaly.client_path + TOML.dump(config) end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index e05be4a3405..8ac73bc8ff2 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -2,10 +2,10 @@ namespace :gitlab do namespace :storage do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do - legacy_projects_count = Project.with_legacy_storage.count + legacy_projects_count = Project.with_unmigrated_storage.count if legacy_projects_count == 0 - puts 'There are no projects using legacy storage. Nothing to do!' + puts 'There are no projects requiring storage migration. Nothing to do!' next end @@ -23,22 +23,42 @@ namespace :gitlab do desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' task legacy_projects: :environment do - projects_summary(Project.with_legacy_storage) + relation_summary('projects', Project.without_storage_feature(:repository)) end desc 'Gitlab | Storage | List existing projects using Legacy Storage' task list_legacy_projects: :environment do - projects_list(Project.with_legacy_storage) + projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository)) end desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' task hashed_projects: :environment do - projects_summary(Project.with_hashed_storage) + relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository)) end desc 'Gitlab | Storage | List existing projects using Hashed Storage' task list_hashed_projects: :environment do - projects_list(Project.with_hashed_storage) + projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository)) + end + + desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage' + task legacy_attachments: :environment do + relation_summary('attachments using Legacy Storage', legacy_attachments_relation) + end + + desc 'Gitlab | Storage | List existing project attachments using Legacy Storage' + task list_legacy_attachments: :environment do + attachments_list('attachments using Legacy Storage', legacy_attachments_relation) + end + + desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage' + task hashed_attachments: :environment do + relation_summary('attachments using Hashed Storage', hashed_attachments_relation) + end + + desc 'Gitlab | Storage | List existing project attachments using Hashed Storage' + task list_hashed_attachments: :environment do + attachments_list('attachments using Hashed Storage', hashed_attachments_relation) end def batch_size @@ -46,29 +66,43 @@ namespace :gitlab do end def project_id_batches(&block) - Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) yield ids.min, ids.max end end - def projects_summary(relation) - projects_count = relation.count - puts "* Found #{projects_count} projects".color(:green) + def legacy_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def hashed_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def relation_summary(relation_name, relation) + relation_count = relation.count + puts "* Found #{relation_count} #{relation_name}".color(:green) - projects_count + relation_count end - def projects_list(relation) - projects_count = projects_summary(relation) + def projects_list(relation_name, relation) + relation_count = relation_summary(relation_name, relation) projects = relation.with_route limit = ENV.fetch('LIMIT', 500).to_i - return unless projects_count > 0 + return unless relation_count > 0 - puts " ! Displaying first #{limit} projects..." if projects_count > limit + puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit counter = 0 projects.find_in_batches(batch_size: batch_size) do |batch| @@ -81,5 +115,26 @@ namespace :gitlab do end end end + + def attachments_list(relation_name, relation) + relation_count = relation_summary(relation_name, relation) + + limit = ENV.fetch('LIMIT', 500).to_i + + return unless relation_count > 0 + + puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit + + counter = 0 + relation.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |upload| + counter += 1 + + puts " - #{upload.path} (id: #{upload.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end end end |