diff options
Diffstat (limited to 'lib')
56 files changed, 857 insertions, 295 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 66b37fd2bcc..621b9dcecd9 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -62,7 +62,7 @@ module API post ":id/repository/commits" do authorize! :push_code, user_project - attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch]) + attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch]) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -140,7 +140,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9919762cd82..6d6ccefe877 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -14,10 +14,15 @@ module API class User < UserBasic expose :created_at - expose :admin?, as: :is_admin expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end + class UserActivity < Grape::Entity + expose :username + expose :last_activity_on + expose :last_activity_on, as: :last_activity_at # Back-compat + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -25,6 +30,7 @@ module API class UserPublic < User expose :last_sign_in_at expose :confirmed_at + expose :last_activity_on expose :email expose :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity @@ -34,8 +40,9 @@ module API expose :external end - class UserWithPrivateToken < UserPublic + class UserWithPrivateDetails < UserPublic expose :private_token + expose :admin?, as: :is_admin end class Email < Grape::Entity diff --git a/lib/api/files.rb b/lib/api/files.rb index 33fc970dc09..e6ea12c5ab7 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -5,7 +5,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -130,7 +130,7 @@ module API authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] != :success render_api_error!(result[:message], 400) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 810e5063996..718f936a1fc 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -60,6 +60,12 @@ module API rescue JSON::ParserError {} end + + def log_user_activity(actor) + commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS + + ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 215bc03d0e9..5b48ee8665f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -40,6 +40,8 @@ module API response = { status: access_status.status, message: access_status.message } if access_status.status + log_user_activity(actor) + # Return the repository full path so that gitlab-shell has it when # handling ssh commands response[:repository_path] = diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 05423c17449..244725bb292 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -35,7 +35,7 @@ module API optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index cb7aec47cf0..e5793fbc5cb 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -33,6 +33,17 @@ module API end end + def find_merge_requests(args = {}) + args = params.merge(args) + + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations + + merge_requests.reorder(args[:order_by] => args[:sort]) + end + params :optional_params_ce do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' @@ -57,23 +68,15 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, type: String, desc: 'Comma-separated list of label names' use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present? - - merge_requests = - case params[:state] - when 'opened' then merge_requests.opened - when 'closed' then merge_requests.closed - when 'merged' then merge_requests.merged - else merge_requests - end + merge_requests = find_merge_requests(project_id: user_project.id) - merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end @@ -197,14 +200,15 @@ module API end put ':id/merge_requests/:merge_request_iid/merge' do merge_request = find_project_merge_request(params[:merge_request_iid]) + merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! unless merge_request.mergeable_state? + not_allowed! unless merge_request.mergeable_state?(skip_ci_check: merge_when_pipeline_succeeds) - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds) if params[:sha] && merge_request.diff_head_sha != params[:sha] render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) @@ -215,7 +219,7 @@ module API should_remove_source_branch: params[:should_remove_source_branch] } - if params[:merge_when_pipeline_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? ::MergeRequests::MergeWhenPipelineSucceedsService .new(merge_request.target_project, current_user, merge_params) .execute(merge_request) diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 53791166c33..87dfd1573a4 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -13,7 +13,7 @@ module API optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" - optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :job_events, type: Boolean, desc: "Trigger hook on job events" optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" @@ -53,7 +53,10 @@ module API use :project_hook_properties end post ":id/hooks" do - hook = user_project.hooks.new(declared_params(include_missing: false)) + hook_params = declared_params(include_missing: false) + hook_params[:build_events] = hook_params.delete(:job_events) { false } + + hook = user_project.hooks.new(hook_params) if hook.save present hook, with: Entities::ProjectHook @@ -74,7 +77,10 @@ module API put ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - if hook.update_attributes(declared_params(include_missing: false)) + update_params = declared_params(include_missing: false) + update_params[:build_events] = update_params.delete(:job_events) if update_params[:job_events] + + if hook.update_attributes(update_params) present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 50842370947..db4b31b55bc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,7 +11,7 @@ module API optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' @@ -103,6 +103,7 @@ module API end post do attrs = declared_params(include_missing: false) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -205,7 +206,7 @@ module API # CE at_least_one_of_ce = [ - :builds_enabled, + :jobs_enabled, :container_registry_enabled, :default_branch, :description, @@ -236,6 +237,8 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success diff --git a/lib/api/session.rb b/lib/api/session.rb index 002ffd1d154..016415c3023 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,7 +1,7 @@ module API class Session < Grape::API desc 'Login to get token' do - success Entities::UserWithPrivateToken + success Entities::UserWithPrivateDetails end params do optional :login, type: String, desc: 'The username' @@ -14,7 +14,7 @@ module API return unauthorized! unless user return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? - present user, with: Entities::UserWithPrivateToken + present user, with: Entities::UserWithPrivateDetails end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index eedc59f8636..40acaebf670 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -39,10 +39,13 @@ module API params do # CE optional :username, type: String, desc: 'Get a single user with a specific username' + optional :extern_uid, type: String, desc: 'Get a single user with a specific external authentication provider UID' + optional :provider, type: String, desc: 'The external provider' optional :search, type: String, desc: 'Search for a username' optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + all_or_none_of :extern_uid, :provider use :pagination end @@ -51,14 +54,17 @@ module API render_api_error!("Not authorized.", 403) end - if params[:username].present? - users = User.where(username: params[:username]) - else - users = User.all - users = users.active if params[:active] - users = users.search(params[:search]) if params[:search].present? - users = users.blocked if params[:blocked] - users = users.external if params[:external] && current_user.admin? + authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) + + users = User.all + users = User.where(username: params[:username]) if params[:username] + users = users.active if params[:active] + users = users.search(params[:search]) if params[:search].present? + users = users.blocked if params[:blocked] + + if current_user.admin? + users = users.joins(:identities).merge(Identity.with_extern_uid(params[:provider], params[:extern_uid])) if params[:extern_uid] && params[:provider] + users = users.external if params[:external] end entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic @@ -427,7 +433,7 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic + present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic end desc "Get the currently authenticated user's SSH keys" do @@ -534,6 +540,21 @@ module API email.destroy current_user.update_secondary_emails! end + + desc 'Get a list of user activities' + params do + optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY' + use :pagination + end + get "activities" do + authenticated_as_admin! + + activities = User. + where(User.arel_table[:last_activity_on].gteq(params[:from])). + reorder(last_activity_on: :asc) + + present paginate(activities), with: Entities::UserActivity + end end end end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 3414a2883e5..674de592f0a 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -53,7 +53,7 @@ module API attrs = declared_params.dup branch = attrs.delete(:branch_name) - attrs.merge!(branch: branch, start_branch: branch, target_branch: branch) + attrs.merge!(start_branch: branch, branch_name: branch) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -131,7 +131,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb index 13542b0c71c..c76acc86504 100644 --- a/lib/api/v3/files.rb +++ b/lib/api/v3/files.rb @@ -6,7 +6,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -123,7 +123,7 @@ module API file_params = declared_params(include_missing: false) file_params[:branch] = file_params.delete(:branch_name) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 4016ac76348..d97e5d98229 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -80,16 +80,32 @@ module Backup 'port' => '--port', 'socket' => '--socket', 'username' => '--user', - 'encoding' => '--default-character-set' + 'encoding' => '--default-character-set', + # SSL + 'sslkey' => '--ssl-key', + 'sslcert' => '--ssl-cert', + 'sslca' => '--ssl-ca', + 'sslcapath' => '--ssl-capath', + 'sslcipher' => '--ssl-cipher' } args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact end def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + args = { + 'username' => 'PGUSER', + 'host' => 'PGHOST', + 'port' => 'PGPORT', + 'password' => 'PGPASSWORD', + # SSL + 'sslmode' => 'PGSSLMODE', + 'sslkey' => 'PGSSLKEY', + 'sslcert' => 'PGSSLCERT', + 'sslrootcert' => 'PGSSLROOTCERT', + 'sslcrl' => 'PGSSLCRL', + 'sslcompression' => 'PGSSLCOMPRESSION' + } + args.each { |opt, arg| ENV[arg] = config[opt].to_s if config[opt] } end def report_success(success) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 7b4476fa4db..8d64c82272a 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -17,9 +17,8 @@ module Backup s[:skipped] = ENV["SKIP"] tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" - Dir.chdir(Gitlab.config.backup.path) do - File.open("#{Gitlab.config.backup.path}/backup_information.yml", - "w+") do |file| + Dir.chdir(backup_path) do + File.open("#{backup_path}/backup_information.yml", "w+") do |file| file << s.to_yaml.gsub(/^---\n/, '') end @@ -64,9 +63,9 @@ module Backup $progress.print "Deleting tmp directories ... " backup_contents.each do |dir| - next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + next unless File.exist?(File.join(backup_path, dir)) - if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) + if FileUtils.rm_rf(File.join(backup_path, dir)) $progress.puts "done".color(:green) else puts "deleting tmp directory '#{dir}' failed".color(:red) @@ -83,8 +82,8 @@ module Backup if keep_time > 0 removed = 0 - Dir.chdir(Gitlab.config.backup.path) do - Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file| + Dir.chdir(backup_path) do + backup_file_list.each do |file| next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ timestamp = $1.to_i @@ -107,18 +106,14 @@ module Backup end def unpack - Dir.chdir(Gitlab.config.backup.path) + Dir.chdir(backup_path) # check for existing backups in the backup dir - file_list = Dir.glob("*#{FILE_NAME_SUFFIX}") - - if file_list.count == 0 - $progress.puts "No backups found in #{Gitlab.config.backup.path}" + if backup_file_list.empty? + $progress.puts "No backups found in #{backup_path}" $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 - end - - if file_list.count > 1 && ENV["BACKUP"].nil? + elsif backup_file_list.many? && ENV["BACKUP"].nil? $progress.puts 'Found more than one backup, please specify which one you want to restore:' $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 @@ -127,7 +122,7 @@ module Backup tar_file = if ENV['BACKUP'].present? "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" else - file_list.first + backup_file_list.first end unless File.exist?(tar_file) @@ -169,6 +164,14 @@ module Backup private + def backup_path + Gitlab.config.backup.path + end + + def backup_file_list + @backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}") + end + def connect_to_remote_directory(connection_settings) connection = ::Fog::Storage.new(connection_settings) diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 6b78aa795b4..327ea9449a1 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -9,12 +9,14 @@ module Banzai VISIBLE_STATES = %w(closed merged).freeze def call + return doc unless context[:issuable_state_filter_enabled] + extractor = Banzai::IssuableExtractor.new(project, current_user) issuables = extractor.extract([doc]) issuables.each do |node, issuable| - if VISIBLE_STATES.include?(issuable.state) - node.children.last.content += " [#{issuable.state}]" + if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + node.content += " (#{issuable.state})" end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index b2537117558..5325819d828 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -7,14 +7,14 @@ module Banzai # class PlantumlFilter < HTML::Pipeline::Filter def call - return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled + return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled plantuml_setup - doc.css('pre.plantuml').each do |el| + doc.css('pre > code[lang="plantuml"]').each do |node| img_tag = Nokogiri::HTML::DocumentFragment.parse( - Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {})) - el.replace img_tag + Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})) + node.parent.replace(img_tag) end doc diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index c5ce360e172..cbabf9156de 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -28,9 +28,13 @@ module Banzai issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) - issue_parser.issues_for_nodes(nodes).merge( + issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( merge_request_parser.merge_requests_for_nodes(nodes) ) + + # The project for the issue/MR might be pending for deletion! + # Filter them out because we don't care about them. + issuables_for_nodes.select { |node, issuable| issuable.project } end end end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index dabf71d6aeb..c2503fa2adc 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -136,7 +136,8 @@ module Banzai nodes.each_with_object({}) do |node, hash| if node.has_attribute?(attribute) - hash[node] = objects_by_id[node.attr(attribute).to_i] + obj = objects_by_id[node.attr(attribute).to_i] + hash[node] = obj if obj end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 74663556cbb..c7801cb5baf 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - module_function - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -16,7 +14,7 @@ module Banzai # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String - def render(text, context = {}) + def self.render(text, context = {}) cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) @@ -35,24 +33,16 @@ module Banzai # of HTML. This method is analogous to calling render(object.field), but it # can cache the rendered HTML in the object, rather than Redis. # - # The context to use is learned from the passed-in object by calling - # #banzai_render_context(field), and cannot be changed. Use #render, passing - # it the field text, if a custom rendering is needed. The generated context - # is returned along with the HTML. - def render_field(object, field) - html_field = object.markdown_cache_field_for(field) - - html = object.__send__(html_field) - return html if html.present? - - html = cacheless_render_field(object, field) - update_object(object, html_field, html) unless object.new_record? || object.destroyed? + # The context to use is managed by the object and cannot be changed. + # Use #render, passing it the field text, if a custom rendering is needed. + def self.render_field(object, field) + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) - html + object.cached_html_for(field) end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field, options = {}) + def self.cacheless_render_field(object, field, options = {}) text = object.__send__(field) context = object.banzai_render_context(field).merge(options) @@ -82,7 +72,7 @@ module Banzai # texts_and_contexts # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] - def cache_collection_render(texts_and_contexts) + def self.cache_collection_render(texts_and_contexts) items_collection = texts_and_contexts.each_with_index do |item, index| context = item[:context] cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) @@ -111,7 +101,7 @@ module Banzai items_collection.map { |item| item[:rendered] } end - def render_result(text, context = {}) + def self.render_result(text, context = {}) text = Pipeline[:pre_process].to_html(text, context) if text Pipeline[context[:pipeline]].call(text, context) @@ -130,7 +120,7 @@ module Banzai # :user - User object # # Returns an HTML-safe String - def post_process(html, context) + def self.post_process(html, context) context = Pipeline[context[:pipeline]].transform_context(context) pipeline = Pipeline[:post_process] @@ -141,7 +131,7 @@ module Banzai end.html_safe end - def cacheless_render(text, context = {}) + def self.cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -154,7 +144,7 @@ module Banzai end end - def full_cache_key(cache_key, pipeline_name) + def self.full_cache_key(cache_key, pipeline_name) return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end @@ -162,13 +152,14 @@ module Banzai # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key # method. - def full_cache_multi_key(cache_key, pipeline_name) + def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end - def update_object(object, html_field, html) - object.update_column(html_field, html) + # GitLab EE needs to disable updates on GET requests in Geo + def self.update_object?(object) + true end end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 1020452480a..b439b0ee29b 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -172,7 +172,7 @@ module Ci close_open_tags() OpenStruct.new( - html: @out, + html: @out.force_encoding(Encoding.default_external), state: state, append: append, truncated: truncated, diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index a4b5f2aba6c..61849a40383 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -15,7 +15,7 @@ module ContainerRegistry LEVELS_SUPPORTED = 3 def initialize(path) - @path = path + @path = path.to_s.downcase end def valid? @@ -25,7 +25,7 @@ module ContainerRegistry end def components - @components ||= @path.to_s.split('/') + @components ||= @path.split('/') end def nodes @@ -48,7 +48,7 @@ module ContainerRegistry end def root_repository? - @path == repository_project.full_path + @path == project_path end def repository_project @@ -60,7 +60,13 @@ module ContainerRegistry def repository_name return unless has_project? - @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?)) + @path.remove(%r(^#{Regexp.escape(project_path)}/?)) + end + + def project_path + return unless has_project? + + repository_project.full_path.downcase end def to_s diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index eee5601b0ed..ea918b23a63 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -108,7 +108,7 @@ module Gitlab token = Doorkeeper::AccessToken.by_token(password) if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) - Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) end end end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 2af94e2c60e..fa462cbe095 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -4,7 +4,7 @@ module Gitlab # This was inspired from: http://stackoverflow.com/a/10219411/1520132 class Stream BUFFER_SIZE = 4096 - LIMIT_SIZE = 50.kilobytes + LIMIT_SIZE = 500.kilobytes attr_reader :stream @@ -14,6 +14,7 @@ module Gitlab def initialize @stream = yield + @stream&.binmode end def valid? @@ -25,11 +26,10 @@ module Gitlab end def limit(last_bytes = LIMIT_SIZE) - stream_size = size - if stream_size < last_bytes - last_bytes = stream_size + if last_bytes < size + stream.seek(-last_bytes, IO::SEEK_END) + stream.readline end - stream.seek(-last_bytes, IO::SEEK_END) end def append(data, offset) @@ -52,7 +52,7 @@ module Gitlab read_last_lines(last_lines) else stream.read - end + end.force_encoding(Encoding.default_external) end def html_with_state(state = nil) @@ -61,8 +61,8 @@ module Gitlab def html(last_lines: nil) text = raw(last_lines: last_lines) - stream = StringIO.new(text) - ::Ci::Ansi2html.convert(stream).html + buffer = StringIO.new(text) + ::Ci::Ansi2html.convert(buffer).html end def extract_coverage(regex) @@ -76,11 +76,14 @@ module Gitlab stream.each_line do |line| matches = line.scan(regex) next unless matches.is_a?(Array) + next if matches.empty? match = matches.flatten.last coverage = match.gsub(/\d+(\.\d+)?/).first - return coverage.to_f if coverage.present? + return coverage if coverage.present? end + + nil rescue # if bad regex or something goes wrong we dont want to interrupt transition # so we just silentrly ignore error for now @@ -111,7 +114,6 @@ module Gitlab end chunks.join.lines.last(last_lines).join - .force_encoding(Encoding.default_external) end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 63b8d0d3b9d..d0bd1299671 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -57,16 +57,16 @@ module Gitlab postgresql? ? "RANDOM()" : "RAND()" end - def true_value - if Gitlab::Database.postgresql? + def self.true_value + if postgresql? "'t'" else 1 end end - def false_value - if Gitlab::Database.postgresql? + def self.false_value + if postgresql? "'f'" else 0 diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 525aa920328..6dabbe0264c 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -89,7 +89,8 @@ module Gitlab ADD CONSTRAINT #{key_name} FOREIGN KEY (#{column}) REFERENCES #{target} (id) - ON DELETE #{on_delete} NOT VALID; + #{on_delete ? "ON DELETE #{on_delete}" : ''} + NOT VALID; EOF # Validate the existing constraint. This can potentially take a very @@ -114,6 +115,14 @@ module Gitlab execute('SET statement_timeout TO 0') if Database.postgresql? end + def true_value + Database.true_value + end + + def false_value + Database.false_value + end + # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. @@ -250,6 +259,245 @@ module Gitlab raise error end end + + # Renames a column without requiring downtime. + # + # Concurrent renames work by using database triggers to ensure both the + # old and new column are in sync. However, this method will _not_ remove + # the triggers or the old column automatically; this needs to be done + # manually in a post-deployment migration. This can be done using the + # method `cleanup_concurrent_column_rename`. + # + # table - The name of the database table containing the column. + # old - The old column name. + # new - The new column name. + # type - The type of the new column. If no type is given the old column's + # type is used. + def rename_column_concurrently(table, old, new, type: nil) + if transaction_open? + raise 'rename_column_concurrently can not be run inside a transaction' + end + + trigger_name = rename_trigger_name(table, old, new) + quoted_table = quote_table_name(table) + quoted_old = quote_column_name(old) + quoted_new = quote_column_name(new) + + if Database.postgresql? + install_rename_triggers_for_postgresql(trigger_name, quoted_table, + quoted_old, quoted_new) + else + install_rename_triggers_for_mysql(trigger_name, quoted_table, + quoted_old, quoted_new) + end + + old_col = column_for(table, old) + new_type = type || old_col.type + + add_column(table, new, new_type, + limit: old_col.limit, + default: old_col.default, + null: old_col.null, + precision: old_col.precision, + scale: old_col.scale) + + update_column_in_batches(table, new, Arel::Table.new(table)[old]) + + copy_indexes(table, old, new) + copy_foreign_keys(table, old, new) + end + + # Changes the type of a column concurrently. + # + # table - The table containing the column. + # column - The name of the column to change. + # new_type - The new column type. + def change_column_type_concurrently(table, column, new_type) + temp_column = "#{column}_for_type_change" + + rename_column_concurrently(table, column, temp_column, type: new_type) + end + + # Performs cleanup of a concurrent type change. + # + # table - The table containing the column. + # column - The name of the column to change. + # new_type - The new column type. + def cleanup_concurrent_column_type_change(table, column) + temp_column = "#{column}_for_type_change" + + transaction do + # This has to be performed in a transaction as otherwise we might have + # inconsistent data. + cleanup_concurrent_column_rename(table, column, temp_column) + rename_column(table, temp_column, column) + end + end + + # Cleans up a concurrent column name. + # + # This method takes care of removing previously installed triggers as well + # as removing the old column. + # + # table - The name of the database table. + # old - The name of the old column. + # new - The name of the new column. + def cleanup_concurrent_column_rename(table, old, new) + trigger_name = rename_trigger_name(table, old, new) + + if Database.postgresql? + remove_rename_triggers_for_postgresql(table, trigger_name) + else + remove_rename_triggers_for_mysql(trigger_name) + end + + remove_column(table, old) + end + + # Performs a concurrent column rename when using PostgreSQL. + def install_rename_triggers_for_postgresql(trigger, table, old, new) + execute <<-EOF.strip_heredoc + CREATE OR REPLACE FUNCTION #{trigger}() + RETURNS trigger AS + $BODY$ + BEGIN + NEW.#{new} := NEW.#{old}; + RETURN NEW; + END; + $BODY$ + LANGUAGE 'plpgsql' + VOLATILE + EOF + + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger} + BEFORE INSERT OR UPDATE + ON #{table} + FOR EACH ROW + EXECUTE PROCEDURE #{trigger}() + EOF + end + + # Installs the triggers necessary to perform a concurrent column rename on + # MySQL. + def install_rename_triggers_for_mysql(trigger, table, old, new) + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger}_insert + BEFORE INSERT + ON #{table} + FOR EACH ROW + SET NEW.#{new} = NEW.#{old} + EOF + + execute <<-EOF.strip_heredoc + CREATE TRIGGER #{trigger}_update + BEFORE UPDATE + ON #{table} + FOR EACH ROW + SET NEW.#{new} = NEW.#{old} + EOF + end + + # Removes the triggers used for renaming a PostgreSQL column concurrently. + def remove_rename_triggers_for_postgresql(table, trigger) + execute("DROP TRIGGER #{trigger} ON #{table}") + execute("DROP FUNCTION #{trigger}()") + end + + # Removes the triggers used for renaming a MySQL column concurrently. + def remove_rename_triggers_for_mysql(trigger) + execute("DROP TRIGGER #{trigger}_insert") + execute("DROP TRIGGER #{trigger}_update") + end + + # Returns the (base) name to use for triggers when renaming columns. + def rename_trigger_name(table, old, new) + 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) + end + + # Returns an Array containing the indexes for the given column + def indexes_for(table, column) + column = column.to_s + + indexes(table).select { |index| index.columns.include?(column) } + end + + # Returns an Array containing the foreign keys for the given column. + def foreign_keys_for(table, column) + column = column.to_s + + foreign_keys(table).select { |fk| fk.column == column } + end + + # Copies all indexes for the old column to a new column. + # + # table - The table containing the columns and indexes. + # old - The old column. + # new - The new column. + def copy_indexes(table, old, new) + old = old.to_s + new = new.to_s + + indexes_for(table, old).each do |index| + new_columns = index.columns.map do |column| + column == old ? new : column + end + + # This is necessary as we can't properly rename indexes such as + # "ci_taggings_idx". + unless index.name.include?(old) + raise "The index #{index.name} can not be copied as it does not "\ + "mention the old column. You have to rename this index manually first." + end + + name = index.name.gsub(old, new) + + options = { + unique: index.unique, + name: name, + length: index.lengths, + order: index.orders + } + + # These options are not supported by MySQL, so we only add them if + # they were previously set. + options[:using] = index.using if index.using + options[:where] = index.where if index.where + + unless index.opclasses.blank? + opclasses = index.opclasses.dup + + # Copy the operator classes for the old column (if any) to the new + # column. + opclasses[new] = opclasses.delete(old) if opclasses[old] + + options[:opclasses] = opclasses + end + + add_concurrent_index(table, new_columns, options) + end + end + + # Copies all foreign keys for the old column to the new column. + # + # table - The table containing the columns and indexes. + # old - The old column. + # new - The new column. + def copy_foreign_keys(table, old, new) + foreign_keys_for(table, old).each do |fk| + add_concurrent_foreign_key(fk.from_table, + fk.to_table, + column: new, + on_delete: fk.on_delete) + end + end + + # Returns the column for the given table and column name. + def column_for(table, name) + name = name.to_s + + columns(table).find { |column| column.name == name } + end end end end diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb new file mode 100644 index 00000000000..7ae5a4c17c8 --- /dev/null +++ b/lib/gitlab/database/multi_threaded_migration.rb @@ -0,0 +1,52 @@ +module Gitlab + module Database + module MultiThreadedMigration + MULTI_THREAD_AR_CONNECTION = :thread_local_ar_connection + + # This overwrites the default connection method so that every thread can + # use a thread-local connection, while still supporting all of Rails' + # migration methods. + def connection + Thread.current[MULTI_THREAD_AR_CONNECTION] || + ActiveRecord::Base.connection + end + + # Starts a thread-pool for N threads, along with N threads each using a + # single connection. The provided block is yielded from inside each + # thread. + # + # Example: + # + # with_multiple_threads(4) do + # execute('SELECT ...') + # end + # + # thread_count - The number of threads to start. + # + # join - When set to true this method will join the threads, blocking the + # caller until all threads have finished running. + # + # Returns an Array containing the started threads. + def with_multiple_threads(thread_count, join: true) + pool = Gitlab::Database.create_connection_pool(thread_count) + + threads = Array.new(thread_count) do + Thread.new do + pool.with_connection do |connection| + begin + Thread.current[MULTI_THREAD_AR_CONNECTION] = connection + yield + ensure + Thread.current[MULTI_THREAD_AR_CONNECTION] = nil + end + end + end + end + + threads.each(&:join) if join + + threads + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 329d12f13d1..0bd226ef050 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -15,6 +15,10 @@ module Gitlab super.tap { |_| store_highlight_cache } end + def real_size + @merge_request_diff.real_size + end + private # Extracted method to highlight in the same iteration to the diff_collection. diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index 4d04f867268..c7542a8fabc 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -82,7 +82,7 @@ module Gitlab file_diff, old_line, new_line = results - Position.new( + new_position = Position.new( old_path: file_diff.old_path, new_path: file_diff.new_path, head_sha: new_diff_refs.head_sha, @@ -91,6 +91,13 @@ module Gitlab old_line: old_line, new_line: new_line ) + + # If a position is found, but is not actually contained in the diff, for example + # because it was an unchanged line in the context of a change that was undone, + # we cannot return this as a successful trace. + return unless new_position.diff_line(repository) + + new_position end private diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 3f6ace0311a..0bba433d04b 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -16,6 +16,10 @@ module Gitlab def execute raise NotImplementedError end + + def metrics_params + { handler: self.class.name } + end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index b8ec9138c10..e7f91607e7e 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,4 +1,3 @@ - require 'gitlab/email/handler/base_handler' module Gitlab @@ -37,6 +36,10 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(project: project) + end + private def create_issue diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 0e22f2189ee..31bb775c357 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -7,6 +7,8 @@ module Gitlab class CreateNoteHandler < BaseHandler include ReplyProcessing + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+\z/ end @@ -26,16 +28,16 @@ module Gitlab record_name: 'comment') end + def metrics_params + super.merge(project: project) + end + private def author sent_notification.recipient end - def project - sent_notification.project - end - def sent_notification @sent_notification ||= SentNotification.for(mail_key) end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 97d7a8d65ff..df70a063330 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -4,6 +4,8 @@ module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ end @@ -17,6 +19,10 @@ module Gitlab noteable.unsubscribe(sent_notification.recipient) end + def metrics_params + super.merge(project: project) + end + private def sent_notification diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ec0529b5a4b..419d56a51e0 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,4 +1,3 @@ - require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver @@ -32,6 +31,8 @@ module Gitlab raise UnknownIncomingEmail unless handler + Gitlab::Metrics.add_event(:receive_email, handler.metrics_params) + handler.execute end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index e56eb0d3beb..98fd4e78126 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -8,7 +8,7 @@ module Gitlab # the user. We load as much as we can for encoding detection # (Linguist) and LFS pointer parsing. All other cases where we need full # blob data should use load_all_data!. - MAX_DATA_DISPLAY_SIZE = 10485760 + MAX_DATA_DISPLAY_SIZE = 10.megabytes attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary @@ -153,7 +153,7 @@ module Gitlab def lfs_size if has_lfs_version_key? size = data.match(/(?<=size )([0-9]+)/) - return size[1] if size + return size[1].to_i if size end nil diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb index e57d228e688..f918074cb14 100644 --- a/lib/gitlab/git/encoding_helper.rb +++ b/lib/gitlab/git/encoding_helper.rb @@ -40,7 +40,13 @@ module Gitlab def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end else clean(message) end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index af1744c9c46..1add037fa5f 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,8 +1,12 @@ module Gitlab module Git class Index + IndexError = Class.new(StandardError) + DEFAULT_MODE = 0o100644 + ACTIONS = %w(create create_dir update move delete).freeze + attr_reader :repository, :raw_index def initialize(repository) @@ -23,9 +27,8 @@ module Gitlab def create(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end add_blob(options) @@ -34,13 +37,12 @@ module Gitlab def create_dir(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end if dir_exists?(options[:file_path]) - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists") + raise IndexError, "A directory with this name already exists" end options = options.dup @@ -55,7 +57,7 @@ module Gitlab file_entry = get(options[:file_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" end add_blob(options, mode: file_entry[:mode]) @@ -66,7 +68,11 @@ module Gitlab file_entry = get(options[:previous_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" + end + + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end raw_index.remove(options[:previous_path]) @@ -77,9 +83,8 @@ module Gitlab def delete(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + unless get(options[:file_path]) + raise IndexError, "A file with this name doesn't exist" end raw_index.remove(options[:file_path]) @@ -95,10 +100,20 @@ module Gitlab end def normalize_path(path) + unless path + raise IndexError, "You must provide a file path" + end + pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - if pathname.each_filename.include?('..') - raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path') + pathname.each_filename do |segment| + if segment == '..' + raise IndexError, 'Path cannot include directory traversal' + end + + unless segment =~ Gitlab::Regex.file_name_regex + raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" + end end pathname.to_s @@ -106,6 +121,10 @@ module Gitlab def add_blob(options, mode: nil) content = options[:content] + unless content + raise IndexError, "You must provide content" + end + content = Base64.decode64(content) if options[:encoding] == 'base64' detect = CharlockHolmes::EncodingDetector.new.detect(content) @@ -119,7 +138,7 @@ module Gitlab raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) rescue Rugged::IndexError => e - raise Gitlab::Git::Repository::InvalidBlobName.new(e.message) + raise IndexError, e.message end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 41ab73abb56..d7dac9f6149 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,13 +45,15 @@ module Gitlab # Default branch in the repository def root_ref - @root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled| - if is_enabled - gitaly_ref_client.default_branch_name - else - discover_default_branch - end - end + # NOTE: This feature is intentionally disabled until + # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved + # @root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled| + # if is_enabled + # gitaly_ref_client.default_branch_name + # else + @root_ref ||= discover_default_branch + # end + # end rescue GRPC::BadStatus => e raise CommandError.new(e) end @@ -70,13 +72,15 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| - if is_enabled - gitaly_ref_client.branch_names - else - branches.map(&:name) - end - end + # Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| + # NOTE: This feature is intentionally disabled until + # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved + # if is_enabled + # gitaly_ref_client.branch_names + # else + branches.map(&:name) + # end + # end rescue GRPC::BadStatus => e raise CommandError.new(e) end @@ -131,13 +135,15 @@ module Gitlab # Returns an Array of tag names def tag_names - Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| - if is_enabled - gitaly_ref_client.tag_names - else - rugged.tags.map { |t| t.name } - end - end + # Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| + # NOTE: This feature is intentionally disabled until + # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved + # if is_enabled + # gitaly_ref_client.tag_names + # else + rugged.tags.map { |t| t.name } + # end + # end rescue GRPC::BadStatus => e raise CommandError.new(e) end @@ -458,17 +464,19 @@ module Gitlab # Returns a RefName for a given SHA def ref_name_for_sha(ref_path, sha) - Gitlab::GitalyClient.migrate(:find_ref_name) do |is_enabled| - if is_enabled - gitaly_ref_client.find_ref_name(sha, ref_path) - else - args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - Gitlab::Popen.popen(args, @path).first.split.last - end - end + # NOTE: This feature is intentionally disabled until + # https://gitlab.com/gitlab-org/gitaly/issues/180 is resolved + # Gitlab::GitalyClient.migrate(:find_ref_name) do |is_enabled| + # if is_enabled + # gitaly_ref_client.find_ref_name(sha, ref_path) + # else + args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + + # Not found -> ["", 0] + # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] + Gitlab::Popen.popen(args, @path).first.split.last + # end + # end end # Returns commits collection diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index bcdf1b1faa8..c69676a1dac 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -15,7 +15,7 @@ module Gitlab end unless URI(address).scheme.in?(%w(tcp unix)) - raise "Unsupported Gitaly address: #{address.inspect}" + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" end @addresses[name] = address diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index b02b9737493..5ca3e6a95ca 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -1,7 +1,23 @@ module Gitlab module GoogleCodeImport class Importer - attr_reader :project, :repo + attr_reader :project, :repo, :closed_statuses + + NICE_LABEL_COLOR_HASH = + { + 'Status: New' => '#428bca', + 'Status: Accepted' => '#5cb85c', + 'Status: Started' => '#8e44ad', + 'Priority: Critical' => '#ffcfcf', + 'Priority: High' => '#deffcf', + 'Priority: Medium' => '#fff5cc', + 'Priority: Low' => '#cfe9ff', + 'Type: Defect' => '#d9534f', + 'Type: Enhancement' => '#44ad8e', + 'Type: Task' => '#4b6dd0', + 'Type: Review' => '#8e44ad', + 'Type: Other' => '#7f8c8d' + }.freeze def initialize(project) @project = project @@ -161,45 +177,19 @@ module Gitlab end def nice_label_color(name) - case name - when /\AComponent:/ - "#fff39e" - when /\AOpSys:/ - "#e2e2e2" - when /\AMilestone:/ - "#fee3ff" - - when "Status: New" - "#428bca" - when "Status: Accepted" - "#5cb85c" - when "Status: Started" - "#8e44ad" - - when "Priority: Critical" - "#ffcfcf" - when "Priority: High" - "#deffcf" - when "Priority: Medium" - "#fff5cc" - when "Priority: Low" - "#cfe9ff" - - when "Type: Defect" - "#d9534f" - when "Type: Enhancement" - "#44ad8e" - when "Type: Task" - "#4b6dd0" - when "Type: Review" - "#8e44ad" - when "Type: Other" - "#7f8c8d" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" - else - "#e2e2e2" - end + NICE_LABEL_COLOR_HASH[name] || + case name + when /\AComponent:/ + '#fff39e' + when /\AOpSys:/ + '#e2e2e2' + when /\AMilestone:/ + '#fee3ff' + when *closed_statuses.map { |s| nice_status_name(s) } + '#cfcfcf' + else + '#e2e2e2' + end end def nice_label_name(name) diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb new file mode 100644 index 00000000000..d392214867a --- /dev/null +++ b/lib/gitlab/issuable_sorter.rb @@ -0,0 +1,29 @@ +module Gitlab + module IssuableSorter + class << self + def sort(project, issuables, &sort_key) + grouped_items = issuables.group_by do |issuable| + if issuable.project.id == project.id + :project_ref + elsif issuable.project.namespace.id == project.namespace.id + :namespace_ref + else + :full_ref + end + end + + natural_sort_issuables(grouped_items[:project_ref], project) + + natural_sort_issuables(grouped_items[:namespace_ref], project) + + natural_sort_issuables(grouped_items[:full_ref], project) + end + + private + + def natural_sort_issuables(issuables, project) + VersionSorter.sort(issuables || []) do |issuable| + issuable.to_reference(project) + end + end + end + end +end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index dda371e6554..49285e35251 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -1,6 +1,11 @@ module Gitlab module MarkupHelper - module_function + extend self + + MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze + ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze + OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -8,10 +13,7 @@ module Gitlab # # Returns boolean def markup?(filename) - gitlab_markdown?(filename) || - asciidoc?(filename) || - filename.downcase.end_with?(*%w(.textile .rdoc .org .creole .wiki - .mediawiki .rst)) + EXTENSIONS.include?(extension(filename)) end # Public: Determines if a given filename is compatible with @@ -21,7 +23,7 @@ module Gitlab # # Returns boolean def gitlab_markdown?(filename) - filename.downcase.end_with?(*%w(.mdown .mkd .mkdn .md .markdown)) + MARKDOWN_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename has AsciiDoc extension. @@ -30,7 +32,7 @@ module Gitlab # # Returns boolean def asciidoc?(filename) - filename.downcase.end_with?(*%w(.adoc .ad .asciidoc)) + ASCIIDOC_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename is plain text. @@ -39,12 +41,17 @@ module Gitlab # # Returns boolean def plain?(filename) - filename.downcase.end_with?('.txt') || - filename.casecmp('readme').zero? + extension(filename) == 'txt' || filename.casecmp('readme').zero? end def previewable?(filename) markup?(filename) end + + private + + def extension(filename) + File.extname(filename).downcase.delete('.') + end end end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 857e0abf710..c6dfa4ad9bd 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -138,6 +138,11 @@ module Gitlab @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' end + # Allow access from other metrics related middlewares + def self.current_transaction + Transaction.current + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -149,10 +154,5 @@ module Gitlab new(udp: { host: host, port: port }) end end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index f98481c6d3a..afd24b4dcc5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -148,7 +148,7 @@ module Gitlab def build_new_user user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) - Users::CreateService.new(nil, user_params).build + Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end def user_attributes diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e599dd4a656..08b061d5e31 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -73,22 +73,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@', '+' and '.'." end - def file_path_regex - @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze - end - - def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." - end - - def directory_traversal_regex - @directory_traversal_regex ||= /\.{2}/.freeze - end - - def directory_traversal_regex_message - "cannot include directory traversal." - end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb new file mode 100644 index 00000000000..6aca6db3123 --- /dev/null +++ b/lib/gitlab/usage_data.rb @@ -0,0 +1,65 @@ +module Gitlab + class UsageData + include Gitlab::CurrentSettings + + class << self + def data(force_refresh: false) + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + end + + def uncached_data + license_usage_data.merge(system_usage_data) + end + + def to_json(force_refresh: false) + data(force_refresh: force_refresh).to_json + end + + def system_usage_data + { + counts: { + boards: Board.count, + ci_builds: ::Ci::Build.count, + ci_pipelines: ::Ci::Pipeline.count, + ci_runners: ::Ci::Runner.count, + ci_triggers: ::Ci::Trigger.count, + deploy_keys: DeployKey.count, + deployments: Deployment.count, + environments: Environment.count, + groups: Group.count, + issues: Issue.count, + keys: Key.count, + labels: Label.count, + lfs_objects: LfsObject.count, + merge_requests: MergeRequest.count, + milestones: Milestone.count, + notes: Note.count, + pages_domains: PagesDomain.count, + projects: Project.count, + projects_prometheus_active: PrometheusService.active.count, + protected_branches: ProtectedBranch.count, + releases: Release.count, + services: Service.where(active: true).count, + snippets: Snippet.count, + todos: Todo.count, + uploads: Upload.count, + web_hooks: WebHook.count + } + } + end + + def license_usage_data + usage_data = { + uuid: current_application_settings.uuid, + version: Gitlab::VERSION, + active_user_count: User.active.count, + recorded_at: Time.now, + mattermost_enabled: Gitlab.config.mattermost.enabled, + edition: 'CE' + } + + usage_data + end + end + end +end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb new file mode 100644 index 00000000000..eb36ab9fded --- /dev/null +++ b/lib/gitlab/user_activities.rb @@ -0,0 +1,34 @@ +module Gitlab + class UserActivities + include Enumerable + + KEY = 'users:activities'.freeze + BATCH_SIZE = 500 + + def self.record(key, time = Time.now) + Gitlab::Redis.with do |redis| + redis.hset(KEY, key, time.to_i) + end + end + + def delete(*keys) + Gitlab::Redis.with do |redis| + redis.hdel(KEY, keys) + end + end + + def each + cursor = 0 + loop do + cursor, pairs = + Gitlab::Redis.with do |redis| + redis.hscan(KEY, cursor, count: BATCH_SIZE) + end + + Hash[pairs].each { |pair| yield pair } + + break if cursor == '0' + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e6e40f6945d..c551f939df1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -168,7 +168,7 @@ module Gitlab end def secret_path - Rails.root.join('.gitlab_workhorse_secret') + Gitlab.config.workhorse.secret_file end def set_key_and_notify(key, value, expire: nil, overwrite: true) diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index d55923673b1..125a3d560d6 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -21,12 +21,7 @@ namespace :cache do end end - desc "GitLab | Clear database cache (in the background)" - task db: :environment do - ClearDatabaseCacheWorker.perform_async - end - - task all: [:db, :redis] + task all: [:redis] end task clear: 'cache:clear:redis' diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a9a48f7188f..f41c73154f5 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -431,8 +431,7 @@ namespace :gitlab do def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user - gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" + puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" Gitlab.config.repositories.storages.each do |name, repository_storage| repo_base_path = repository_storage['path'] @@ -443,15 +442,16 @@ namespace :gitlab do break end - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + user_id = uid_for(gitlab_shell_ssh_user) + root_group_id = gid_for('root') + group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)] + if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid) puts "yes".color(:green) else puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + puts " User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue) try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}" ) for_more_information( see_installation_guide_section "GitLab Shell" diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 9f6cfe3957c..046780481ba 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -2,20 +2,23 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" task :install, [:dir] => :environment do |t, args| + require 'toml' + warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end - tag = "v#{Gitlab::GitalyClient.expected_server_version}" + version = Gitlab::GitalyClient.expected_server_version repo = 'https://gitlab.com/gitlab-org/gitaly.git' - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' Dir.chdir(args.dir) do + create_gitaly_configuration run_command!([command]) end end @@ -33,5 +36,39 @@ namespace :gitlab do puts TOML.dump(storage: config) end + + private + + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only write a config.toml file in the most common and simplest case: the + # case where we have exactly one Gitaly process and we are sure it is + # running locally because it uses a Unix socket. + def create_gitaly_configuration + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + File.open("config.toml", "w") do |f| + f.puts TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storages: storages) + end + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end end end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index dd2fda54e62..95687066819 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -1,19 +1,18 @@ namespace :gitlab do namespace :shell do desc "GitLab | Install or upgrade gitlab-shell" - task :install, [:tag, :repo] => :environment do |t, args| + task :install, [:repo] => :environment do |t, args| warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = "v#{default_version}" - args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) + checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index cdba2262bc2..e3c9d3b491c 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -147,41 +147,30 @@ module Gitlab Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home end - def checkout_or_clone_tag(tag:, repo:, target_dir:) - if Dir.exist?(target_dir) - checkout_tag(tag, target_dir) - else - clone_repo(repo, target_dir) - end + def checkout_or_clone_version(version:, repo:, target_dir:) + version = + if version.starts_with?("=") + version.sub(/\A=/, '') # tag or branch + else + "v#{version}" # tag + end - reset_to_tag(tag, target_dir) + clone_repo(repo, target_dir) unless Dir.exist?(target_dir) + checkout_version(version, target_dir) + reset_to_version(version, target_dir) end def clone_repo(repo, target_dir) run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) end - def checkout_tag(tag, target_dir) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + def checkout_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}]) end - def reset_to_tag(tag_wanted, target_dir) - tag = - begin - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) - rescue Gitlab::TaskFailedError - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) - end - - if tag - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) - else - raise Gitlab::TaskFailedError - end + def reset_to_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}]) end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index dbdfb335a5c..cb2adc81c9d 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -44,7 +44,7 @@ namespace :gitlab do ), Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/ + /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ ) ].freeze diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index baea94bf8ca..a00b02188cf 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -7,10 +7,10 @@ namespace :gitlab do abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end - tag = "v#{Gitlab::Workhorse.version}" + version = Gitlab::Workhorse.version repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 15131fbf755..a9dad6a1bf0 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -52,7 +52,6 @@ class NewImporter < ::Gitlab::GithubImport::Importer project.repository.add_remote(project.import_type, project.import_url) project.repository.set_remote_as_mirror(project.import_type) project.repository.fetch_remote(project.import_type, forced: true) - project.repository.remove_remote(project.import_type) rescue => e # Expire cache to prevent scenarios such as: # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true |