diff options
Diffstat (limited to 'lib')
80 files changed, 606 insertions, 544 deletions
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b7aadc27e71..6769855b899 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -112,9 +112,9 @@ module API can_push = params[:can_push].nil? ? deploy_keys_project.can_push : params[:can_push] title = params[:title] || deploy_keys_project.deploy_key.title - result = deploy_keys_project.update_attributes(can_push: can_push, - deploy_key_attributes: { id: params[:key_id], - title: title }) + result = deploy_keys_project.update(can_push: can_push, + deploy_key_attributes: { id: params[:key_id], + title: title }) if result present deploy_keys_project, with: Entities::DeployKeysProject diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 3a6e707fd5b..40df1e79bc7 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -532,6 +532,12 @@ module API end class MergeRequestBasic < ProjectEntity + expose :title_html, if: -> (_, options) { options[:render_html] } do |entity| + MarkupHelper.markdown_field(entity, :title) + end + expose :description_html, if: -> (_, options) { options[:render_html] } do |entity| + MarkupHelper.markdown_field(entity, :description) + end expose :target_branch, :source_branch expose :upvotes do |merge_request, options| if options[:issuable_metadata] diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 0f46bc4c98e..2621c9f8fc2 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -232,6 +232,7 @@ module API params do requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' + optional :render_html, type: Boolean, desc: 'Returns the description and title rendered HTML' end desc 'Get a single merge request' do success Entities::MergeRequest @@ -239,7 +240,7 @@ module API get ':id/merge_requests/:merge_request_iid' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project, render_html: params[:render_html] end desc 'Get the participants of a merge request' do diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 5ef4e9d530c..15c57a2fc02 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -23,9 +23,13 @@ module API get ':id/export/download' do path = user_project.export_project_path - render_api_error!('404 Not found or has expired', 404) unless path - - present_disk_file!(path, File.basename(path), 'application/gzip') + if path + present_disk_file!(path, File.basename(path), 'application/gzip') + elsif user_project.export_project_object_exists? + present_carrierwave_file!(user_project.import_export_upload.export_file) + else + render_api_error!('404 Not found or has expired', 404) + end end desc 'Start export' do diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 68921ae439b..4760a1c08d7 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -80,7 +80,7 @@ module API update_params = declared_params(include_missing: false) - if hook.update_attributes(update_params) + if hook.update(update_params) present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? diff --git a/lib/api/services.rb b/lib/api/services.rb index 794fdab8f2b..553e8dff4b9 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -787,7 +787,7 @@ module API service = user_project.find_or_initialize_service(service_slug.underscore) service_params = declared_params(include_missing: false).merge(active: true) - if service.update_attributes(service_params) + if service.update(service_params) present service, with: Entities::ProjectService else render_api_error!('400 Bad Request', 400) @@ -807,7 +807,7 @@ module API hash.merge!(key => nil) end - unless service.update_attributes(attrs.merge(active: false)) + unless service.update(attrs.merge(active: false)) render_api_error!('400 Bad Request', 400) end end diff --git a/lib/api/users.rb b/lib/api/users.rb index e8df2c5a74a..5aaaf104dff 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -186,7 +186,7 @@ module API identity = user.identities.find_by(provider: identity_attrs[:provider]) if identity - identity.update_attributes(identity_attrs) + identity.update(identity_attrs) else identity = user.identities.build(identity_attrs) identity.save diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 60a12dca9d3..b39b11009b3 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -100,6 +100,11 @@ module Banzai ref_pattern = object_class.reference_pattern link_pattern = object_class.link_reference_pattern + # Compile often used regexps only once outside of the loop + ref_pattern_anchor = /\A#{ref_pattern}\z/ + link_pattern_start = /\A#{link_pattern}/ + link_pattern_anchor = /\A#{link_pattern}\z/ + nodes.each do |node| if text_node?(node) && ref_pattern replace_text_when_pattern_matches(node, ref_pattern) do |content| @@ -108,7 +113,7 @@ module Banzai elsif element_node?(node) yield_valid_link(node) do |link, inner_html| - if ref_pattern && link =~ /\A#{ref_pattern}\z/ + if ref_pattern && link =~ ref_pattern_anchor replace_link_node_with_href(node, link) do object_link_filter(link, ref_pattern, link_content: inner_html) end @@ -118,7 +123,7 @@ module Banzai next unless link_pattern - if link == inner_html && inner_html =~ /\A#{link_pattern}/ + if link == inner_html && inner_html =~ link_pattern_start replace_link_node_with_text(node, link) do object_link_filter(inner_html, link_pattern, link_reference: true) end @@ -126,7 +131,7 @@ module Banzai next end - if link =~ /\A#{link_pattern}\z/ + if link =~ link_pattern_anchor replace_link_node_with_href(node, link) do object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true) end diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index bc9597df894..dbb25280849 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -18,7 +18,7 @@ module Banzai PARSE_OPTIONS = [ :FOOTNOTES, # parse footnotes. :STRIKETHROUGH_DOUBLE_TILDE, # parse strikethroughs by double tildes (as redcarpet does). - :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. + :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. ].freeze # The `:GITHUB_PRE_LANG` option is not used intentionally because diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index 47542194497..da3fabba39b 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -119,8 +119,8 @@ module DeclarativePolicy # a PolicyDsl which is used for registering the rule with # this class. PolicyDsl will call back into Base.enable_when, # Base.prevent_when, and Base.prevent_all_when. - def rule(&b) - rule = RuleDsl.new(self).instance_eval(&b) + def rule(&block) + rule = RuleDsl.new(self).instance_eval(&block) PolicyDsl.new(self, rule) end @@ -222,8 +222,8 @@ module DeclarativePolicy # computes the given ability and prints a helpful debugging output # showing which - def debug(ability, *a) - runner(ability).debug(*a) + def debug(ability, *args) + runner(ability).debug(*args) end desc "Unknown user" @@ -274,7 +274,7 @@ module DeclarativePolicy # # NOTE we can't use ||= here because the value might be the # boolean `false` - def cache(key, &b) + def cache(key) return @cache[key] if cached?(key) @cache[key] = yield diff --git a/lib/declarative_policy/delegate_dsl.rb b/lib/declarative_policy/delegate_dsl.rb index f544dffe888..ca2eb98e3e8 100644 --- a/lib/declarative_policy/delegate_dsl.rb +++ b/lib/declarative_policy/delegate_dsl.rb @@ -7,10 +7,10 @@ module DeclarativePolicy @delegate_name = delegate_name end - def method_missing(m, *a, &b) - return super unless a.empty? && !block_given? + def method_missing(msg, *args) + return super unless args.empty? && !block_given? - @rule_dsl.delegate(@delegate_name, m) + @rule_dsl.delegate(@delegate_name, msg) end end end diff --git a/lib/declarative_policy/policy_dsl.rb b/lib/declarative_policy/policy_dsl.rb index f11b6e9f730..c96049768a1 100644 --- a/lib/declarative_policy/policy_dsl.rb +++ b/lib/declarative_policy/policy_dsl.rb @@ -15,8 +15,8 @@ module DeclarativePolicy @rule = rule end - def policy(&b) - instance_eval(&b) + def policy(&block) + instance_eval(&block) end def enable(*abilities) @@ -31,14 +31,14 @@ module DeclarativePolicy @context_class.prevent_all_when(@rule) end - def method_missing(m, *a, &b) - return super unless @context_class.respond_to?(m) + def method_missing(msg, *args, &block) + return super unless @context_class.respond_to?(msg) - @context_class.__send__(m, *a, &b) # rubocop:disable GitlabSecurity/PublicSend + @context_class.__send__(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end - def respond_to_missing?(m) - @context_class.respond_to?(m) || super + def respond_to_missing?(msg) + @context_class.respond_to?(msg) || super end end end diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index 5c214408dd0..c77784cb49d 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -2,7 +2,7 @@ module DeclarativePolicy # rubocop:disable Naming/FileName PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope" class << self - def with_preferred_scope(scope, &b) + def with_preferred_scope(scope) Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY] yield ensure @@ -13,12 +13,12 @@ module DeclarativePolicy # rubocop:disable Naming/FileName Thread.current[PREFERRED_SCOPE_KEY] end - def user_scope(&b) - with_preferred_scope(:user, &b) + def user_scope(&block) + with_preferred_scope(:user, &block) end - def subject_scope(&b) - with_preferred_scope(:subject, &b) + def subject_scope(&block) + with_preferred_scope(:subject, &block) end def preferred_scope=(scope) diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index e309244a3b3..407398cc770 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -8,8 +8,8 @@ module DeclarativePolicy # how that affects the actual ability decision - for that, a # `Step` is used. class Base - def self.make(*a) - new(*a).simplify + def self.make(*args) + new(*args).simplify end # true or false whether this rule passes. diff --git a/lib/declarative_policy/rule_dsl.rb b/lib/declarative_policy/rule_dsl.rb index e948b7f2de1..7254b08eda5 100644 --- a/lib/declarative_policy/rule_dsl.rb +++ b/lib/declarative_policy/rule_dsl.rb @@ -32,13 +32,13 @@ module DeclarativePolicy Rule::DelegatedCondition.new(delegate_name, condition) end - def method_missing(m, *a, &b) - return super unless a.empty? && !block_given? + def method_missing(msg, *args) + return super unless args.empty? && !block_given? - if @context_class.delegations.key?(m) - DelegateDsl.new(self, m) + if @context_class.delegations.key?(msg) + DelegateDsl.new(self, msg) else - cond(m.to_sym) + cond(msg.to_sym) end end end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 87f14b3b0d2..fec672f4b8c 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -127,7 +127,7 @@ module DeclarativePolicy # # For each step, we yield the step object along with the computed score # for debugging purposes. - def steps_by_score(&b) + def steps_by_score flatten_steps! if @steps.size > 50 diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index a9b04c183ad..e8dbde176ef 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -139,6 +139,11 @@ module ExtractsPath def lfs_blob_ids blob_ids = tree.blobs.map(&:id) + + # When current endpoint is a Blob then `tree.blobs` will be empty, it means we need to analyze + # the current Blob in order to determine if it's a LFS object + blob_ids = Array.wrap(@repo.blob_at(@commit.id, @path)&.id) if blob_ids.empty? # rubocop:disable Gitlab/ModuleWithInstanceVariables + @lfs_blob_ids = Gitlab::Git::Blob.batch_lfs_pointers(@project.repository, blob_ids).map(&:id) # rubocop:disable Gitlab/ModuleWithInstanceVariables end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index b9a148f35bf..ab6b609d099 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -9,10 +9,6 @@ module Gitlab Settings end - def self.migrations_hash - @_migrations_hash ||= Digest::MD5.hexdigest(ActiveRecord::Migrator.get_all_versions.to_s) - end - def self.revision @_revision ||= begin if File.exist?(root.join("REVISION")) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index e7283b2f9e8..589e8062226 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -48,7 +48,7 @@ module Gitlab gl_user rescue ActiveRecord::RecordInvalid => e log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" - return self, e.record.errors + [self, e.record.errors] end def gl_user diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index d5cf9e0d53a..cb2bdea755c 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Style/Documentation -# rubocop:disable Metrics/LineLength module Gitlab module BackgroundMigration diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 5a4e5b2c471..92096e29ef1 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 1b4a9e8a194..ccd1f9b4dba 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb index c2bf42f846d..da8265a3a5f 100644 --- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb index 0b785e1b056..8fb2c334048 100644 --- a/lib/gitlab/background_migration/delete_diff_files.rb +++ b/lib/gitlab/background_migration/delete_diff_files.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index a357538a885..58df74cfa9b 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/LineLength # rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb index 22b0ac71920..103bd98af14 100644 --- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb index d0816ae3ed5..77c1f1ffaf0 100644 --- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb index 94c65459a67..cba3e21cea6 100644 --- a/lib/gitlab/background_migration/fill_store_upload.rb +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb new file mode 100644 index 00000000000..0a12401c35f --- /dev/null +++ b/lib/gitlab/background_migration/fix_cross_project_label_links.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FixCrossProjectLabelLinks + GROUP_NESTED_LEVEL = 10.freeze + + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + class Label < ActiveRecord::Base + self.inheritance_column = :_type_disabled + self.table_name = 'labels' + end + + class LabelLink < ActiveRecord::Base + self.table_name = 'label_links' + end + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + end + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + end + + class Namespace < ActiveRecord::Base + self.inheritance_column = :_type_disabled + self.table_name = 'namespaces' + + def self.groups_with_descendants_ids(start_id, stop_id) + # To isolate migration code, we avoid usage of + # Gitlab::GroupHierarchy#base_and_descendants which already + # does this job better + ids = Namespace.where(type: 'Group', id: Label.where(type: 'GroupLabel').select('distinct group_id')).where(id: start_id..stop_id).pluck(:id) + group_ids = ids + + GROUP_NESTED_LEVEL.times do + ids = Namespace.where(type: 'Group', parent_id: ids).pluck(:id) + break if ids.empty? + + group_ids += ids + end + + group_ids.uniq + end + end + + def perform(start_id, stop_id) + group_ids = Namespace.groups_with_descendants_ids(start_id, stop_id) + project_ids = Project.where(namespace_id: group_ids).select(:id) + + fix_issues(project_ids) + fix_merge_requests(project_ids) + end + + private + + # select IDs of issues which reference a label which is: + # a) a project label of a different project, or + # b) a group label of a different group than issue's project group + def fix_issues(project_ids) + issue_ids = Label + .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'Issue\' + INNER JOIN issues ON issues.id = label_links.target_id + INNER JOIN projects ON projects.id = issues.project_id') + .where('issues.project_id in (?)', project_ids) + .where('(labels.project_id is not null and labels.project_id != issues.project_id) '\ + 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)') + .select('distinct issues.id') + + Issue.where(id: issue_ids).find_each { |issue| check_resource_labels(issue, issue.project_id) } + end + + # select IDs of MRs which reference a label which is: + # a) a project label of a different project, or + # b) a group label of a different group than MR's project group + def fix_merge_requests(project_ids) + mr_ids = Label + .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'MergeRequest\' + INNER JOIN merge_requests ON merge_requests.id = label_links.target_id + INNER JOIN projects ON projects.id = merge_requests.target_project_id') + .where('merge_requests.target_project_id in (?)', project_ids) + .where('(labels.project_id is not null and labels.project_id != merge_requests.target_project_id) '\ + 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)') + .select('distinct merge_requests.id') + + MergeRequest.where(id: mr_ids).find_each { |merge_request| check_resource_labels(merge_request, merge_request.target_project_id) } + end + + def check_resource_labels(resource, project_id) + local_labels = available_labels(project_id) + + # get all label links for the given resource (issue/MR) + # which reference a label not included in avaiable_labels + # (other than its project labels and labels of ancestor groups) + cross_labels = LabelLink + .select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id') + .joins('INNER JOIN labels ON labels.id = label_links.label_id') + .where(target_type: resource.class.name.demodulize, target_id: resource.id) + .where('labels.id not in (?)', local_labels.select(:id)) + + cross_labels.each do |label| + matching_label = local_labels.find {|l| l.title == label.title && l.color == label.color} + + next unless matching_label + + Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" + LabelLink.update(label.label_link_id, label_id: matching_label.id) + end + end + + # get all labels available for the project (including + # group labels of ancestor groups) + def available_labels(project_id) + @labels ||= {} + @labels[project_id] ||= Label + .where("(type = 'GroupLabel' and group_id in (?)) or (type = 'ProjectLabel' and id = ?)", + project_group_ids(project_id), + project_id) + end + + def project_group_ids(project_id) + ids = [Project.find(project_id).namespace_id] + + GROUP_NESTED_LEVEL.times do + group = Namespace.find(ids.last) + break unless group.parent_id + + ids << group.parent_id + end + + ids + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb index 242e3143e71..268c6083d3c 100644 --- a/lib/gitlab/background_migration/migrate_build_stage.rb +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 7088aa0860a..38fecac1bfe 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb index 7f243073fd0..ef50fe4adb1 100644 --- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb index a4ef51fd0e8..5b2b2af718a 100644 --- a/lib/gitlab/background_migration/move_personal_snippet_files.rb +++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb index d9d3d2e667b..698f5e46c0c 100644 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/LineLength # rubocop:disable Metrics/ClassLength # rubocop:disable Metrics/BlockLength # rubocop:disable Style/Documentation diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb index a976cb4c243..aa4f130538c 100644 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -19,7 +19,7 @@ module Gitlab create_fork_networks_for_missing_projects(start_id, end_id) create_fork_networks_memberships_for_root_projects(start_id, end_id) - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY BackgroundMigrationWorker.perform_in( delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] ) diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb index 8a901a9bf39..d89ce358bb9 100644 --- a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb +++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb @@ -1,7 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength -# rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/ClassLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 9232f20a063..a19dc9747fb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -4,7 +4,7 @@ module Gitlab module BackgroundMigration # This class processes a batch of rows in `untracked_files_for_uploads` by # adding each file to the `uploads` table if it does not exist. - class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength + class PopulateUntrackedUploads def perform(start_id, end_id) return unless migrate? diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index a2c5acbde71..4a9a62aaeb5 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -4,7 +4,7 @@ module Gitlab module PopulateUntrackedUploadsDependencies # This class is responsible for producing the attributes necessary to # track an uploaded file in the `uploads` table. - class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength + class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename @@ -134,7 +134,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength + upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR base = %r{\A#{Regexp.escape(upload_dir)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 522c69a0bb1..81ca2b0a9b7 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -144,7 +144,7 @@ module Gitlab def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') "#{UntrackedFile.table_name} (path) VALUES #{values}" diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 35eadf6fa93..e780f8c646b 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -29,105 +29,105 @@ module Gitlab end class Converter - def on_0(s) reset() end + def on_0(_) reset() end - def on_1(s) enable(STYLE_SWITCHES[:bold]) end + def on_1(_) enable(STYLE_SWITCHES[:bold]) end - def on_3(s) enable(STYLE_SWITCHES[:italic]) end + def on_3(_) enable(STYLE_SWITCHES[:italic]) end - def on_4(s) enable(STYLE_SWITCHES[:underline]) end + def on_4(_) enable(STYLE_SWITCHES[:underline]) end - def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + def on_8(_) enable(STYLE_SWITCHES[:conceal]) end - def on_9(s) enable(STYLE_SWITCHES[:cross]) end + def on_9(_) enable(STYLE_SWITCHES[:cross]) end - def on_21(s) disable(STYLE_SWITCHES[:bold]) end + def on_21(_) disable(STYLE_SWITCHES[:bold]) end - def on_22(s) disable(STYLE_SWITCHES[:bold]) end + def on_22(_) disable(STYLE_SWITCHES[:bold]) end - def on_23(s) disable(STYLE_SWITCHES[:italic]) end + def on_23(_) disable(STYLE_SWITCHES[:italic]) end - def on_24(s) disable(STYLE_SWITCHES[:underline]) end + def on_24(_) disable(STYLE_SWITCHES[:underline]) end - def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + def on_28(_) disable(STYLE_SWITCHES[:conceal]) end - def on_29(s) disable(STYLE_SWITCHES[:cross]) end + def on_29(_) disable(STYLE_SWITCHES[:cross]) end - def on_30(s) set_fg_color(0) end + def on_30(_) set_fg_color(0) end - def on_31(s) set_fg_color(1) end + def on_31(_) set_fg_color(1) end - def on_32(s) set_fg_color(2) end + def on_32(_) set_fg_color(2) end - def on_33(s) set_fg_color(3) end + def on_33(_) set_fg_color(3) end - def on_34(s) set_fg_color(4) end + def on_34(_) set_fg_color(4) end - def on_35(s) set_fg_color(5) end + def on_35(_) set_fg_color(5) end - def on_36(s) set_fg_color(6) end + def on_36(_) set_fg_color(6) end - def on_37(s) set_fg_color(7) end + def on_37(_) set_fg_color(7) end - def on_38(s) set_fg_color_256(s) end + def on_38(stack) set_fg_color_256(stack) end - def on_39(s) set_fg_color(9) end + def on_39(_) set_fg_color(9) end - def on_40(s) set_bg_color(0) end + def on_40(_) set_bg_color(0) end - def on_41(s) set_bg_color(1) end + def on_41(_) set_bg_color(1) end - def on_42(s) set_bg_color(2) end + def on_42(_) set_bg_color(2) end - def on_43(s) set_bg_color(3) end + def on_43(_) set_bg_color(3) end - def on_44(s) set_bg_color(4) end + def on_44(_) set_bg_color(4) end - def on_45(s) set_bg_color(5) end + def on_45(_) set_bg_color(5) end - def on_46(s) set_bg_color(6) end + def on_46(_) set_bg_color(6) end - def on_47(s) set_bg_color(7) end + def on_47(_) set_bg_color(7) end - def on_48(s) set_bg_color_256(s) end + def on_48(stack) set_bg_color_256(stack) end - def on_49(s) set_bg_color(9) end + def on_49(_) set_bg_color(9) end - def on_90(s) set_fg_color(0, 'l') end + def on_90(_) set_fg_color(0, 'l') end - def on_91(s) set_fg_color(1, 'l') end + def on_91(_) set_fg_color(1, 'l') end - def on_92(s) set_fg_color(2, 'l') end + def on_92(_) set_fg_color(2, 'l') end - def on_93(s) set_fg_color(3, 'l') end + def on_93(_) set_fg_color(3, 'l') end - def on_94(s) set_fg_color(4, 'l') end + def on_94(_) set_fg_color(4, 'l') end - def on_95(s) set_fg_color(5, 'l') end + def on_95(_) set_fg_color(5, 'l') end - def on_96(s) set_fg_color(6, 'l') end + def on_96(_) set_fg_color(6, 'l') end - def on_97(s) set_fg_color(7, 'l') end + def on_97(_) set_fg_color(7, 'l') end - def on_99(s) set_fg_color(9, 'l') end + def on_99(_) set_fg_color(9, 'l') end - def on_100(s) set_bg_color(0, 'l') end + def on_100(_) set_bg_color(0, 'l') end - def on_101(s) set_bg_color(1, 'l') end + def on_101(_) set_bg_color(1, 'l') end - def on_102(s) set_bg_color(2, 'l') end + def on_102(_) set_bg_color(2, 'l') end - def on_103(s) set_bg_color(3, 'l') end + def on_103(_) set_bg_color(3, 'l') end - def on_104(s) set_bg_color(4, 'l') end + def on_104(_) set_bg_color(4, 'l') end - def on_105(s) set_bg_color(5, 'l') end + def on_105(_) set_bg_color(5, 'l') end - def on_106(s) set_bg_color(6, 'l') end + def on_106(_) set_bg_color(6, 'l') end - def on_107(s) set_bg_color(7, 'l') end + def on_107(_) set_bg_color(7, 'l') end - def on_109(s) set_bg_color(9, 'l') end + def on_109(_) set_bg_color(9, 'l') end attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask @@ -188,19 +188,19 @@ module Gitlab ) end - def handle_section(s) - action = s[1] - timestamp = s[2] - section = s[3] - line = s.matched()[0...-5] # strips \r\033[0K + def handle_section(scanner) + action = scanner[1] + timestamp = scanner[2] + section = scanner[3] + line = scanner.matched()[0...-5] # strips \r\033[0K @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} end - def handle_sequence(s) - indicator = s[1] - commands = s[2].split ';' - terminator = s[3] + def handle_sequence(scanner) + indicator = scanner[1] + commands = scanner[2].split ';' + terminator = scanner[3] # We are only interested in color and text style changes - triggered by # sequences starting with '\e[' and ending with 'm'. Any other control diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index db47c2f6185..7cddd2c7b7e 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -47,7 +47,7 @@ module Gitlab Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] end - private # rubocop:disable Lint/UselessAccessModifier + private def entry(key, entry, metadata) factory = Entry::Factory.new(entry) diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index a52d71225bb..ee54b893598 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -6,6 +6,7 @@ module Gitlab LEASE_TIMEOUT = 1.hour ArchiveError = Class.new(StandardError) + AlreadyArchivedError = Class.new(StandardError) attr_reader :job @@ -81,7 +82,9 @@ module Gitlab def write(mode) stream = Gitlab::Ci::Trace::Stream.new do - if current_path + if trace_artifact + raise AlreadyArchivedError, 'Could not write to the archived trace' + elsif current_path File.open(current_path, mode) elsif Feature.enabled?('ci_enable_live_trace') Gitlab::Ci::Trace::ChunkedIO.new(job) @@ -98,14 +101,17 @@ module Gitlab end def erase! - trace_artifact&.destroy - - paths.each do |trace_path| - FileUtils.rm(trace_path, force: true) - end - - job.trace_chunks.fast_destroy_all - job.erase_old_trace! + ## + # Erase the archived trace + trace_artifact&.destroy! + + ## + # Erase the live trace + job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace + FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace + job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace + ensure + @current_path = nil end def archive! @@ -117,7 +123,7 @@ module Gitlab private def unsafe_archive! - raise ArchiveError, 'Already archived' if trace_artifact + raise AlreadyArchivedError, 'Could not archive again' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? if job.trace_chunks.any? diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb index 9bb0166c9e3..c09089d6475 100644 --- a/lib/gitlab/ci/trace/section_parser.rb +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -75,19 +75,19 @@ module Gitlab @beginning_of_section_regex ||= /section_/.freeze end - def find_next_marker(s) + def find_next_marker(scanner) beginning_of_section_len = 8 - maybe_marker = s.exist?(beginning_of_section_regex) + maybe_marker = scanner.exist?(beginning_of_section_regex) if maybe_marker.nil? - s.terminate + scanner.terminate else # repositioning at the beginning of the match - s.pos += maybe_marker - beginning_of_section_len + scanner.pos += maybe_marker - beginning_of_section_len if block_given? - good_marker = yield(s) + good_marker = yield(scanner) # if not a good marker: Consuming the matched beginning_of_section_regex - s.pos += beginning_of_section_len unless good_marker + scanner.pos += beginning_of_section_len unless good_marker end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 3cf35f499cd..9147ef401da 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -60,7 +60,7 @@ module Gitlab def in_memory_application_settings with_fallback_to_fake_application_settings do - @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults # rubocop:disable Gitlab/ModuleWithInstanceVariables + @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 4ad106e7b0a..872e70f9a5d 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -42,6 +42,21 @@ module Gitlab !self.read_only? end + # check whether the underlying database is in read-only mode + def self.db_read_only? + if postgresql? + ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') == 't' + else + false + end + end + + def self.db_read_write? + !self.db_read_only? + end + def self.version @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb index 65332dfd239..1f157354ea4 100644 --- a/lib/gitlab/diff/image_point.rb +++ b/lib/gitlab/diff/image_point.rb @@ -3,11 +3,11 @@ module Gitlab class ImagePoint attr_reader :width, :height, :x, :y - def initialize(width, height, x, y) + def initialize(width, height, new_x, new_y) @width = width @height = height - @x = x - @y = y + @x = new_x + @y = new_y end def to_h diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 54783a07919..99970779c67 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -93,7 +93,7 @@ module Gitlab private - def longest_common_prefix(a, b) + def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName max_length = [a.length, b.length].max length = 0 @@ -109,7 +109,7 @@ module Gitlab length end - def longest_common_suffix(a, b) + def longest_common_suffix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName longest_common_prefix(a.reverse, b.reverse) end end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 0b8f6cfe3cb..d1fd5dfe0cb 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -65,17 +65,17 @@ module Gitlab clean(message) end rescue ArgumentError - return nil + nil end - def encode_binary(s) - return "" if s.nil? + def encode_binary(str) + return "" if str.nil? - s.dup.force_encoding(Encoding::ASCII_8BIT) + str.dup.force_encoding(Encoding::ASCII_8BIT) end - def binary_stringio(s) - StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } + def binary_stringio(str) + StringIO.new(str || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end private diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb new file mode 100644 index 00000000000..e998548cff9 --- /dev/null +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -0,0 +1,29 @@ +module Gitlab + # This module provides helper methods which are intregrated with GitLab::ExclusiveLease + module ExclusiveLeaseHelpers + FailedToObtainLockError = Class.new(StandardError) + + ## + # This helper method blocks a process/thread until the other process cancel the obrainted lease key. + # + # Note: It's basically discouraged to use this method in the unicorn's thread, + # because it holds the connection until all `retries` is consumed. + # This could potentially eat up all connection pools. + def in_lock(key, ttl: 1.minute, retries: 10, sleep_sec: 0.01.seconds) + lease = Gitlab::ExclusiveLease.new(key, timeout: ttl) + + until uuid = lease.try_obtain + # Keep trying until we obtain the lease. To prevent hammering Redis too + # much we'll wait for a bit. + sleep(sleep_sec) + break if (retries -= 1) < 0 + end + + raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid + + yield + ensure + Gitlab::ExclusiveLease.cancel(key, uuid) + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 8953bc8c148..a91de278cf3 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -191,19 +191,19 @@ module Gitlab end end - def linkify_issues(s) - s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') - s = s.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') - s + def linkify_issues(str) + str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') + str = str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') + str end - def escape_for_markdown(s) - s = s.gsub(/^#/, "\\#") - s = s.gsub(/^-/, "\\-") - s = s.gsub("`", "\\~") - s = s.delete("\r") - s = s.gsub("\n", " \n") - s + def escape_for_markdown(str) + str = str.gsub(/^#/, "\\#") + str = str.gsub(/^-/, "\\-") + str = str.gsub("`", "\\~") + str = str.delete("\r") + str = str.gsub("\n", " \n") + str end def format_content(raw_content) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 604bb11e712..96fa94d5790 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -50,13 +50,7 @@ module Gitlab end def raw(repository, sha) - Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| - if is_enabled - repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) - else - rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) - end - end + repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) end # Returns an array of Blob instances, specified in blob_references as @@ -207,16 +201,7 @@ module Gitlab return if @loaded_all_data - @data = Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| - begin - if is_enabled - repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data - else - repository.lookup(id).content - end - end - end - + @data = repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data @loaded_all_data = true @loaded_size = @data.bytesize end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index c67826da1d2..4e2d817d12c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -63,12 +63,8 @@ module Gitlab # This saves us an RPC round trip. return nil if commit_id.include?(':') - commit = repo.gitaly_migrate(:find_commit) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.find_commit(commit_id) - else - rugged_find(repo, commit_id) - end + commit = repo.wrapped_gitaly_errors do + repo.gitaly_commit_client.find_commit(commit_id) end decorate(repo, commit) if commit @@ -78,12 +74,6 @@ module Gitlab nil end - def rugged_find(repo, commit_id) - obj = repo.rev_parse_target(commit_id) - - obj.is_a?(Rugged::Commit) ? obj : nil - end - # Get last commit for HEAD # # Ex. @@ -174,13 +164,8 @@ module Gitlab # relation to each other. The last 10 commits for a branch for example, # should go through .where def batch_by_oid(repo, oids) - repo.gitaly_migrate(:list_commits_by_oid, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.list_commits_by_oid(oids) - else - oids.map { |oid| find(repo, oid) }.compact - end + repo.wrapped_gitaly_errors do + repo.gitaly_commit_client.list_commits_by_oid(oids) end end @@ -299,14 +284,7 @@ module Gitlab def deltas @deltas ||= begin - deltas = Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.commit_deltas(self) - else - rugged_diff_from_parent.each_delta - end - end - + deltas = @repository.gitaly_commit_client.commit_deltas(self) deltas.map { |delta| Gitlab::Git::Diff.new(delta) } end end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index c3cb0264112..0e4a973301f 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -12,14 +12,8 @@ module Gitlab end def conflicts - @conflicts ||= begin - @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| - if is_enabled - gitaly_conflicts_client(@target_repository).list_conflict_files.to_a - else - rugged_list_conflict_files - end - end + @conflicts ||= @target_repository.wrapped_gitaly_errors do + gitaly_conflicts_client(@target_repository).list_conflict_files.to_a end rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) @@ -28,12 +22,8 @@ module Gitlab end def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) - source_repository.gitaly_migrate(:conflicts_resolve_conflicts) do |is_enabled| - if is_enabled - gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) - else - rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) - end + source_repository.wrapped_gitaly_errors do + gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) end end @@ -61,57 +51,6 @@ module Gitlab def gitaly_conflicts_client(repository) repository.gitaly_conflicts_client(@our_commit_oid, @their_commit_oid) end - - def write_resolved_file_to_index(repository, index, file, params) - if params[:sections] - resolved_lines = file.resolve_lines(params[:sections]) - new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") - - new_file << "\n" if file.our_blob.data.end_with?("\n") - elsif params[:content] - new_file = file.resolve_content(params[:content]) - end - - our_path = file.our_path - - oid = repository.rugged.write(new_file, :blob) - index.add(path: our_path, oid: oid, mode: file.our_mode) - index.conflict_remove(our_path) - end - - def rugged_list_conflict_files - target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - conflict_files(@target_repository, target_index) - end - - def rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) - source_repository.with_repo_branch_commit(@target_repository, target_branch) do - index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - conflicts = conflict_files(source_repository, index) - - resolution.files.each do |file_params| - conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) - - write_resolved_file_to_index(source_repository, index, conflict_file, file_params) - end - - unless index.conflicts.empty? - missing_files = index.conflicts.map { |file| file[:ours][:path] } - - raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" - end - - commit_params = { - message: resolution.commit_message, - parents: [@our_commit_oid, @their_commit_oid] - } - - source_repository.commit_index(resolution.user, source_branch, index, commit_params) - end - end end end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 6a601561c2a..219c69893ad 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -42,12 +42,10 @@ module Gitlab return if @overflow return if @iterator.nil? - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled && @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher) - each_gitaly_patch(&block) - else - each_rugged_patch(&block) - end + if @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher) + each_gitaly_patch(&block) + else + each_serialized_patch(&block) end @populated = true @@ -118,7 +116,7 @@ module Gitlab end end - def each_rugged_patch + def each_serialized_patch i = @array.length @iterator.each do |raw| diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index bbfe6ab1d95..76404366e8e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -439,31 +439,11 @@ module Gitlab raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") end - gitaly_migrate(:find_commits) do |is_enabled| - if is_enabled - gitaly_commit_client.find_commits(options) - else - raw_log(options).map { |c| Commit.decorate(self, c) } - end + wrapped_gitaly_errors do + gitaly_commit_client.find_commits(options) end end - # Used in gitaly-ruby - def raw_log(options) - sha = - unless options[:all] - actual_ref = options[:ref] || root_ref - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - end - - log_by_shell(sha, options) - end - def count_commits(options) options = process_count_commits_options(options.dup) @@ -555,13 +535,7 @@ module Gitlab # diff options. The +options+ hash can also include :break_rewrites to # split larger rewrites into delete/add pairs. def diff(from, to, options = {}, *paths) - iterator = gitaly_migrate(:diff_between) do |is_enabled| - if is_enabled - gitaly_commit_client.diff(from, to, options.merge(paths: paths)) - else - diff_patches(from, to, options, *paths) - end - end + iterator = gitaly_commit_client.diff(from, to, options.merge(paths: paths)) Gitlab::Git::DiffCollection.new(iterator, options) end @@ -651,7 +625,13 @@ module Gitlab end def update_branch(branch_name, user:, newrev:, oldrev:) - OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) + gitaly_migrate(:operation_user_update_branch) do |is_enabled| + if is_enabled + gitaly_operations_client.user_update_branch(branch_name, user, newrev, oldrev) + else + OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) + end + end end def rm_branch(branch_name, user:) @@ -1095,7 +1075,6 @@ module Gitlab true end - # rubocop:disable Metrics/ParameterLists def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, @@ -1107,7 +1086,6 @@ module Gitlab start_branch_name, start_repository) end end - # rubocop:enable Metrics/ParameterLists def write_config(full_path:) return unless full_path.present? @@ -1115,8 +1093,18 @@ module Gitlab # This guard avoids Gitaly log/error spam raise NoRepository, 'repository does not exist' unless exists? + set_config('gitlab.fullpath' => full_path) + end + + def set_config(entries) wrapped_gitaly_errors do - gitaly_repository_client.write_config(full_path: full_path) + gitaly_repository_client.set_config(entries) + end + end + + def delete_config(*keys) + wrapped_gitaly_errors do + gitaly_repository_client.delete_config(keys) end end @@ -1219,12 +1207,10 @@ module Gitlab end def find_commits_by_message(query, ref, path, limit, offset) - gitaly_migrate(:commits_by_message) do |is_enabled| - if is_enabled - find_commits_by_message_by_gitaly(query, ref, path, limit, offset) - else - find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) - end + wrapped_gitaly_errors do + gitaly_commit_client + .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset) + .map { |c| commit(c) } end end @@ -1234,12 +1220,8 @@ module Gitlab end def last_commit_for_path(sha, path) - gitaly_migrate(:last_commit_for_path) do |is_enabled| - if is_enabled - last_commit_for_path_by_gitaly(sha, path) - else - last_commit_for_path_by_rugged(sha, path) - end + wrapped_gitaly_errors do + gitaly_commit_client.last_commit_for_path(sha, path) end end @@ -1450,46 +1432,6 @@ module Gitlab end end - # Gitaly note: JV: although #log_by_shell shells out to Git I think the - # complexity is such that we should migrate it as Ruby before trying to - # do it in Go. - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %w[log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - - if options[:all] - cmd += %w[--all --reverse] - else - cmd << sha - end - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output, _status = run_git(cmd) - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } - end - # We are trying to deprecate this method because it does a lot of work # but it seems to be used only to look up submodule URL's. # https://gitlab.com/gitlab-org/gitaly/issues/329 @@ -1619,11 +1561,6 @@ module Gitlab end end - def last_commit_for_path_by_rugged(sha, path) - sha = last_commit_id_for_path_by_shelling_out(sha, path) - commit(sha) - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -1816,35 +1753,6 @@ module Gitlab raise CommandError, @gitlab_projects.output end - def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) - ref ||= root_ref - - args = %W( - log #{ref} --pretty=%H --skip #{offset} - --max-count #{limit} --grep=#{query} --regexp-ignore-case - ) - args = args.concat(%W(-- #{path})) if path.present? - - git_log_results = run_git(args).first.lines - - git_log_results.map { |c| commit(c.chomp) }.compact - end - - def find_commits_by_message_by_gitaly(query, ref, path, limit, offset) - gitaly_commit_client - .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset) - .map { |c| commit(c) } - end - - def last_commit_for_path_by_gitaly(sha, path) - gitaly_commit_client.last_commit_for_path(sha, path) - end - - def last_commit_id_for_path_by_shelling_out(sha, path) - args = %W(rev-list --max-count=1 #{sha} -- #{path}) - run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip - end - def rugged_merge_base(from, to) rugged.merge_base(from, to) rescue Rugged::ReferenceError @@ -1858,35 +1766,6 @@ module Gitlab def sha_from_ref(ref) rev_parse_target(ref).oid end - - def build_git_cmd(*args) - object_directories = alternate_object_directories.join(File::PATH_SEPARATOR) - - env = { 'PWD' => self.path } - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories if object_directories.present? - - [ - env, - ::Gitlab.config.git.bin_path, - *args, - { chdir: self.path } - ] - end - - def git_diff_cmd(old_rev, new_rev) - old_rev = old_rev == ::Gitlab::Git::BLANK_SHA ? ::Gitlab::Git::EMPTY_TREE_ID : old_rev - - build_git_cmd('diff', old_rev, new_rev, '--raw') - end - - def git_cat_file_cmd - format = '%(objectname) %(objectsize) %(rest)' - build_git_cmd('cat-file', "--batch-check=#{format}") - end - - def format_git_cat_file_script - File.expand_path('../support/format-git-cat-file-input', __FILE__) - end end end end diff --git a/lib/gitlab/git/support/format-git-cat-file-input b/lib/gitlab/git/support/format-git-cat-file-input deleted file mode 100755 index 2e93c646d0f..00000000000 --- a/lib/gitlab/git/support/format-git-cat-file-input +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env ruby - -# This script formats the output of the `git diff <old_rev> <new_rev> --raw` -# command so it can be processed by `git cat-file` - -# We need to convert this: -# ":100644 100644 5f53439... 85bc2f9... R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee" -# To: -# "85bc2f9 R\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee" - -ARGF.each do |line| - _, _, old_blob_id, new_blob_id, rest = line.split(/\s/, 5) - - old_blob_id.gsub!(/[^\h]/, '') - new_blob_id.gsub!(/[^\h]/, '') - - # We can't pass '0000000...' to `git cat-file` given it will not return info about the deleted file - blob_id = new_blob_id =~ /\A0+\z/ ? old_blob_id : new_blob_id - - $stdout.puts "#{blob_id} #{rest}" -end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 66e781a8e5b..58a4060cc96 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -401,8 +401,8 @@ module Gitlab path.read.chomp end - def self.timestamp(t) - Google::Protobuf::Timestamp.new(seconds: t.to_i) + def self.timestamp(time) + Google::Protobuf::Timestamp.new(seconds: time.to_i) end # The default timeout on all Gitaly calls diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 72e1e59d8df..6a97cd8ed17 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -399,8 +399,8 @@ module Gitlab end end - def encode_repeated(a) - Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) + def encode_repeated(array) + Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) } ) end def call_find_commit(revision) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index ab2c61f6782..555733d1834 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -68,6 +68,22 @@ module Gitlab raise Gitlab::Git::Repository::InvalidRef, ex end + def user_update_branch(branch_name, user, newrev, oldrev) + request = Gitaly::UserUpdateBranchRequest.new( + repository: @gitaly_repo, + branch_name: encode_binary(branch_name), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + newrev: encode_binary(newrev), + oldrev: encode_binary(oldrev) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::PreReceiveError, pre_receive_error + end + end + def user_delete_branch(branch_name, user) request = Gitaly::UserDeleteBranchRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 982f8d0963b..64b9af4d70c 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -265,17 +265,39 @@ module Gitlab true end - def write_config(full_path:) - request = Gitaly::WriteConfigRequest.new(repository: @gitaly_repo, full_path: full_path) - response = GitalyClient.call( + def set_config(entries) + return if entries.empty? + + request = Gitaly::SetConfigRequest.new(repository: @gitaly_repo) + entries.each do |key, value| + request.entries << build_set_config_entry(key, value) + end + + GitalyClient.call( + @storage, + :repository_service, + :set_config, + request, + timeout: GitalyClient.fast_timeout + ) + + nil + end + + def delete_config(keys) + return if keys.empty? + + request = Gitaly::DeleteConfigRequest.new(repository: @gitaly_repo, keys: keys) + + GitalyClient.call( @storage, :repository_service, - :write_config, + :delete_config, request, timeout: GitalyClient.fast_timeout ) - raise Gitlab::Git::OSError.new(response.error) unless response.error.empty? + nil end def license_short_name @@ -352,6 +374,23 @@ module Gitlab timeout: timeout ) end + + def build_set_config_entry(key, value) + entry = Gitaly::SetConfigRequest::Entry.new(key: key) + + case value + when String + entry.value_str = value + when Integer + entry.value_int32 = value + when TrueClass, FalseClass + entry.value_bool = value + else + raise InvalidArgument, "invalid git config value: #{value.inspect}" + end + + entry + end end end end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 02fcb413abd..8e530de174d 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -60,8 +60,8 @@ module Gitlab private - def method_missing(m, *args, &block) - @hash.public_send(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + def method_missing(msg, *args, &block) + @hash.public_send(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 46b49128140..5070f4e3cfe 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -200,27 +200,27 @@ module Gitlab "Status: #{name}" end - def linkify_issues(s) - s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') - s = s.gsub(/([Cc]omment) #([0-9]+)/, '\1 \2') - s + def linkify_issues(str) + str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') + str = str.gsub(/([Cc]omment) #([0-9]+)/, '\1 \2') + str end - def escape_for_markdown(s) + def escape_for_markdown(str) # No headings and lists - s = s.gsub(/^#/, "\\#") - s = s.gsub(/^-/, "\\-") + str = str.gsub(/^#/, "\\#") + str = str.gsub(/^-/, "\\-") # No inline code - s = s.gsub("`", "\\`") + str = str.gsub("`", "\\`") # Carriage returns make me sad - s = s.delete("\r") + str = str.delete("\r") # Markdown ignores single newlines, but we need them as <br />. - s = s.gsub("\n", " \n") + str = str.gsub("\n", " \n") - s + str end def create_label(name) diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 6d2278d0876..2716834f566 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -39,7 +39,7 @@ module Gitlab def update_signature!(cached_signature) using_keychain do |gpg_key| - cached_signature.update_attributes!(attributes(gpg_key)) + cached_signature.update!(attributes(gpg_key)) end @signature = cached_signature diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 53fe2f8e436..be3710c5b7f 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -40,6 +40,10 @@ module Gitlab "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end + def object_storage? + Feature.enabled?(:import_export_object_storage) + end + def version VERSION end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index aef371d81eb..83134bb0769 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -2,6 +2,7 @@ module Gitlab module ImportExport module AfterExportStrategies class BaseAfterExportStrategy + extend Gitlab::ImportExport::CommandLineUtil include ActiveModel::Validations extend Forwardable @@ -24,9 +25,10 @@ module Gitlab end def execute(current_user, project) - return unless project&.export_project_path - @project = project + + return unless @project.export_status == :finished + @current_user = current_user if invalid? @@ -51,9 +53,12 @@ module Gitlab end def self.lock_file_path(project) - return unless project&.export_path + return unless project.export_path || object_storage? - File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME) + lock_path = project.import_export_shared.archive_path + + mkdir_p(lock_path) + File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME) end protected @@ -77,6 +82,10 @@ module Gitlab def log_validation_errors errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end + + def object_storage? + project.export_project_object_exists? + end end end end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index 938664a95a1..dce8f89c0ab 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -38,14 +38,20 @@ module Gitlab private def send_file - export_file = File.open(project.export_project_path) - - Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend + Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend ensure - export_file.close if export_file + export_file.close if export_file && !object_storage? + end + + def export_file + if object_storage? + project.import_export_upload.export_file.file.open + else + File.open(project.export_project_path) + end end - def send_file_options(export_file) + def send_file_options { body_stream: export_file, headers: headers @@ -53,7 +59,15 @@ module Gitlab end def headers - { 'Content-Length' => File.size(project.export_project_path).to_s } + { 'Content-Length' => export_size.to_s } + end + + def export_size + if object_storage? + project.import_export_upload.export_file.file.size + else + File.size(project.export_project_path) + end end end end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 2daeba90a51..3cd153a4fd2 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -15,15 +15,22 @@ module Gitlab def save if compress_and_save remove_export_path + Rails.logger.info("Saved project export #{archive_file}") - archive_file + + save_on_object_storage if use_object_storage? else - @shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}")) + @shared.error(Gitlab::ImportExport::Error.new(error_message)) false end rescue => e @shared.error(e) false + ensure + if use_object_storage? + remove_archive + remove_export_path + end end private @@ -36,9 +43,29 @@ module Gitlab FileUtils.rm_rf(@shared.export_path) end + def remove_archive + FileUtils.rm_rf(@shared.archive_path) + end + def archive_file @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) end + + def save_on_object_storage + upload = ImportExportUpload.find_or_initialize_by(project: @project) + + File.open(archive_file) { |file| upload.export_file = file } + + upload.save! + end + + def use_object_storage? + Gitlab::ImportExport.object_storage? + end + + def error_message + "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}" + end end end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index 3d778da90c7..f9ebe53d6af 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -18,7 +18,7 @@ module Gitlab ALPINE_VERSION=$(cat /etc/alpine-release | cut -d '.' -f 1,2) echo http://mirror.clarkson.edu/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories echo http://mirror1.hs-esslingen.de/pub/Mirrors/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories - apk add -U ca-certificates openssl >/dev/null + apk add -U wget ca-certificates openssl >/dev/null wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null mv /tmp/linux-amd64/helm /usr/bin/ HEREDOC diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 344784c866f..db04356a5e9 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -53,7 +53,7 @@ module Gitlab end def config_file - ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../config/gitlab.yml', __dir__) end end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 66f30e3b397..04135dac4ff 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -162,7 +162,6 @@ module Gitlab # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. - # rubocop:disable Gitlab/ModuleWithInstanceVariables def pool if influx_metrics_enabled? if @pool.nil? @@ -180,7 +179,6 @@ module Gitlab @pool end end - # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index b11520a79bb..f3290e3149c 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -1,5 +1,3 @@ -# rubocop:disable Style/ClassVars - module Gitlab module Metrics # Class for tracking timing information about method calls diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 9753be6d5c3..18f91db98fc 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -84,7 +84,7 @@ module Gitlab def open_file(params, key) ::UploadedFile.from_params( params, key, - Gitlab.config.uploads.storage_path) + [FileUploader.root, Gitlab.config.uploads.storage_path]) end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 4a99b7cca5c..8dca431c005 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -69,6 +69,7 @@ module Gitlab @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} end + # Overridden in EE module def whitelisted_routes grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 35ed3a5ac05..a71acda8701 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -1,5 +1,10 @@ module Gitlab class OmniauthInitializer + def self.enabled? + Gitlab.config.omniauth.enabled || + Gitlab.config.omniauth.auto_sign_in_with_provider.present? + end + def initialize(devise_config) @devise_config = devise_config end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 5cedd9e84c2..e222541992a 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -74,17 +74,10 @@ module Gitlab relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') - gitaly_migrate(:create_repository, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - repository = Gitlab::Git::Repository.new(storage, relative_path, '') - repository.gitaly_repository_client.create_repository - true - else - repo_path = File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, relative_path) - Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) - end - end + repository = Gitlab::Git::Repository.new(storage, relative_path, '') + wrapped_gitaly_errors { repository.gitaly_repository_client.create_repository } + + true rescue => err # Once the Rugged codes gets removes this can be improved Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") false @@ -448,7 +441,11 @@ module Gitlab end def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) + wrapped_gitaly_errors { Gitlab::GitalyClient.migrate(method, status: status, &block) } + end + + def wrapped_gitaly_errors + yield rescue GRPC::NotFound, GRPC::BadStatus => e # Old Popen code returns [Error, output] to the caller, so we # need to do the same here... diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e893e46ee86..55c899912f9 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -37,21 +37,14 @@ module Gitlab end def send_git_blob(repository, blob) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) - { - 'GitalyServer' => gitaly_server_hash(repository), - 'GetBlobRequest' => { - repository: repository.gitaly_repository.to_h, - oid: blob.id, - limit: -1 - } - } - else - { - 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id - } - end + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetBlobRequest' => { + repository: repository.gitaly_repository.to_h, + oid: blob.id, + limit: -1 + } + } [ SEND_DATA_HEADER, @@ -91,16 +84,12 @@ module Gitlab end def send_git_diff(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) - { - 'GitalyServer' => gitaly_server_hash(repository), - 'RawDiffRequest' => Gitaly::RawDiffRequest.new( - gitaly_diff_or_patch_hash(repository, diff_refs) - ).to_json - } - else - workhorse_diff_or_patch_hash(repository, diff_refs) - end + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawDiffRequest' => Gitaly::RawDiffRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } [ SEND_DATA_HEADER, diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index be0d97370d0..e877ab10248 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -11,7 +11,7 @@ module Rouge @tag = tag end - def stream(tokens, &b) + def stream(tokens) is_first = true token_lines(tokens) do |line| yield "\n" unless is_first diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 21998dd2f5b..6df7fe81437 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -19,6 +19,23 @@ namespace :gettext do Rake::Task['gettext:po_to_json'].invoke end + task :regenerate do + # Remove all translated files, this speeds up finding + FileUtils.rm Dir['locale/**/gitlab.*'] + # remove the `pot` file to ensure it's completely regenerated + FileUtils.rm_f 'locale/gitlab.pot' + + Rake::Task['gettext:find'].invoke + + # leave only the required changes. + `git checkout -- locale/*/gitlab.po` + + puts <<~MSG + All done. Please commit the changes to `locale/gitlab.pot`. + + MSG + end + desc 'Lint all po files in `locale/' task lint: :environment do require 'simple_po_parser' @@ -50,13 +67,12 @@ namespace :gettext do end task :updated_check do + pot_file = 'locale/gitlab.pot' # Removing all pre-translated files speeds up `gettext:find` as the # files don't need to be merged. # Having `LC_MESSAGES/gitlab.mo files present also confuses the output. FileUtils.rm Dir['locale/**/gitlab.*'] - - # Make sure we start out with a clean pot.file - `git checkout -- locale/gitlab.pot` + FileUtils.rm_f pot_file # `gettext:find` writes touches to temp files to `stderr` which would cause # `static-analysis` to report failures. We can ignore these. @@ -64,18 +80,18 @@ namespace :gettext do Rake::Task['gettext:find'].invoke end - pot_diff = `git diff -- locale/gitlab.pot`.strip + pot_diff = `git diff -- #{pot_file} | grep -E '^(\\+|-)msgid'`.strip # reset the locale folder for potential next tasks `git checkout -- locale` if pot_diff.present? raise <<~MSG - Newly translated strings found, please add them to `gitlab.pot` by running: + Newly translated strings found, please add them to `#{pot_file}` by running: - rm locale/**/gitlab.*; bin/rake gettext:find; git checkout -- locale/*/gitlab.po + bin/rake gettext:regenerate - Then commit and push the resulting changes to `locale/gitlab.pot`. + Then commit and push the resulting changes to `#{pot_file}`. The diff was: diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 78e18992a8e..f548a266b99 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -8,7 +8,7 @@ namespace :gitlab do @uploader_class = args.uploader_class.constantize @model_class = args.model_class.constantize - uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) end def enqueue_batch(batch, index) diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 5dc85b2baea..4b9cb59eab5 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -28,7 +28,7 @@ class UploadedFile @tempfile = File.new(path, 'rb') end - def self.from_params(params, field, upload_path) + def self.from_params(params, field, upload_paths) unless params["#{field}.path"] raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"] @@ -37,7 +37,8 @@ class UploadedFile file_path = File.realpath(params["#{field}.path"]) - unless self.allowed_path?(file_path, [upload_path, Dir.tmpdir].compact) + paths = Array(upload_paths) << Dir.tmpdir + unless self.allowed_path?(file_path, paths.compact) raise InvalidPathError, "insecure path used '#{file_path}'" end |