diff options
Diffstat (limited to 'lib')
39 files changed, 794 insertions, 394 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index b615703df93..6d827448994 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -54,43 +54,25 @@ module API not_found!('Branch') unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) - developers_can_merge = to_boolean(params[:developers_can_merge]) - developers_can_push = to_boolean(params[:developers_can_push]) - protected_branch_params = { - name: @branch.name + name: @branch.name, + developers_can_push: to_boolean(params[:developers_can_push]), + developers_can_merge: to_boolean(params[:developers_can_merge]) } - # If `developers_can_merge` is switched off, _all_ `DEVELOPER` - # merge_access_levels need to be deleted. - if developers_can_merge == false - protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + service_args = [user_project, current_user, protected_branch_params] - # If `developers_can_push` is switched off, _all_ `DEVELOPER` - # push_access_levels need to be deleted. - if developers_can_push == false - protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + protected_branch = if protected_branch + ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + else + ProtectedBranches::ApiCreateService.new(*service_args).execute + end - protected_branch_params.merge!( - merge_access_levels_attributes: [{ - access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }], - push_access_levels_attributes: [{ - access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }] - ) - - if protected_branch - service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params) - service.execute(protected_branch) + if protected_branch.valid? + present @branch, with: Entities::RepoBranch, project: user_project else - service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params) - service.execute + render_api_error!(protected_branch.errors.full_messages, 422) end - - present @branch, with: Entities::RepoBranch, project: user_project end # Unprotect a single branch @@ -123,7 +105,7 @@ module API post ":id/repository/branches" do authorize_push_project result = CreateBranchService.new(user_project, current_user). - execute(params[:branch_name], params[:ref]) + execute(params[:branch_name], params[:ref]) if result[:status] == :success present result[:branch], @@ -142,10 +124,10 @@ module API # Example Request: # DELETE /projects/:id/repository/branches/:branch delete ":id/repository/branches/:branch", - requirements: { branch: /.+/ } do + requirements: { branch: /.+/ } do authorize_push_project result = DeleteBranchService.new(user_project, current_user). - execute(params[:branch]) + execute(params[:branch]) if result[:status] == :success { diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 52bdbcae5a8..67adca6605f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -3,15 +3,32 @@ module API class Builds < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project builds - # - # Parameters: - # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/builds + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: ['pending', 'running', 'failed', 'success', 'canceled'], + coerce_with: ->(scope) { + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + } + end + end + + desc 'Get a project builds' do + success Entities::Build + end + params do + use :optional_scope + end get ':id/builds' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -20,15 +37,13 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get builds for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The SHA id of a commit - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/repository/commits/:sha/builds + desc 'Get builds for a specific commit of a project' do + success Entities::Build + end + params do + requires :sha, type: String, desc: 'The SHA id of a commit' + use :optional_scope + end get ':id/repository/commits/:sha/builds' do authorize_read_builds! @@ -42,13 +57,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/builds/:build_id + desc 'Get a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id' do authorize_read_builds! @@ -58,13 +72,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Download the artifacts file from build - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Example Request: - # GET /projects/:id/builds/:build_id/artifacts + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/artifacts' do authorize_read_builds! @@ -73,14 +86,13 @@ module API present_artifacts!(build.artifacts_file) end - # Download the artifacts file from ref_name and job - # - # Parameters: - # id (required) - The ID of a project - # ref_name (required) - The ref from repository - # job (required) - The name for the build - # Example Request: - # GET /projects/:id/builds/artifacts/:ref_name/download?job=name + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the build' + end get ':id/builds/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_read_builds! @@ -91,17 +103,13 @@ module API present_artifacts!(latest_build.artifacts_file) end - # Get a trace of a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/build/:build_id/trace - # # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific build of a project' + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/trace' do authorize_read_builds! @@ -115,13 +123,12 @@ module API body trace end - # Cancel a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/cancel + desc 'Cancel a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/cancel' do authorize_update_builds! @@ -133,13 +140,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Retry a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/retry + desc 'Retry a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/retry' do authorize_update_builds! @@ -152,13 +158,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Erase build (remove artifacts and build trace) - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example Request: - # post /projects/:id/build/:build_id/erase + desc 'Erase build (remove artifacts and build trace)' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/erase' do authorize_update_builds! @@ -170,13 +175,12 @@ module API user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end - # Keep the artifacts to prevent them from being deleted - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - The ID of a build - # Example Request: - # POST /projects/:id/builds/:build_id/artifacts/keep + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/artifacts/keep' do authorize_update_builds! @@ -235,14 +239,6 @@ module API return builds if scope.nil? || scope.empty? available_statuses = ::CommitStatus::AVAILABLE_STATUSES - scope = - if scope.is_a?(String) - [scope] - elsif scope.is_a?(Hashie::Mash) - scope.values - else - ['unknown'] - end unknown = scope - available_statuses render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 617a240318a..2f2cf769481 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -19,6 +19,7 @@ module API optional :until, type: String, desc: 'Only commits before or in this date will be returned' optional :page, type: Integer, default: 0, desc: 'The page for pagination' optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :path, type: String, desc: 'The file path' end get ":id/repository/commits" do # TODO remove the next line for 9.0, use DateTime type in the params block @@ -28,6 +29,7 @@ module API offset = params[:page] * params[:per_page] commits = user_project.repository.commits(ref, + path: params[:path], limit: params[:per_page], offset: offset, after: params[:since], diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 825e05fbae3..425df2c176a 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -49,18 +49,23 @@ module API attrs = attributes_for_keys [:title, :key] attrs[:key].strip! if attrs[:key] + # Check for an existing key joined to this project key = user_project.deploy_keys.find_by(key: attrs[:key]) - present key, with: Entities::SSHKey if key + if key + present key, with: Entities::SSHKey + break + end # Check for available deploy keys in other projects key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) if key user_project.deploy_keys << key present key, with: Entities::SSHKey + break end + # Create a new deploy key key = DeployKey.new attrs - if key.valid? && user_project.deploy_keys << key present key, with: Entities::SSHKey else diff --git a/lib/api/entities.rb b/lib/api/entities.rb index feaa0c213bf..ab9d2d54f4b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -138,7 +138,7 @@ module API expose :name expose :commit do |repo_branch, options| - options[:project].repository.commit(repo_branch.target) + options[:project].repository.commit(repo_branch.dereferenced_target) end expose :protected do |repo_branch, options| @@ -523,7 +523,7 @@ module API expose :name, :message expose :commit do |repo_tag, options| - options[:project].repository.commit(repo_tag.target) + options[:project].repository.commit(repo_tag.dereferenced_target) end expose :release, using: Entities::Release do |repo_tag, options| diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 45120898b76..8025581d3ca 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -6,6 +6,7 @@ module API SUDO_PARAM = :sudo def to_boolean(value) + return value if [true, false].include?(value) return true if value =~ /^(true|t|yes|y|1|on)$/i return false if value =~ /^(false|f|no|n|0|off)$/i diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9a5d1ece070..ccf181402f9 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -17,15 +17,20 @@ module API # helpers do + def project_path + @project_path ||= begin + project_path = params[:project].sub(/\.git\z/, '') + Repository.remove_storage_from_path(project_path) + end + end + def wiki? - @wiki ||= params[:project].end_with?('.wiki') && - !Project.find_with_namespace(params[:project]) + @wiki ||= project_path.end_with?('.wiki') && + !Project.find_with_namespace(project_path) end def project @project ||= begin - project_path = params[:project] - # Check for *.wiki repositories. # Strip out the .wiki from the pathname before finding the # project. This applies the correct project permissions to diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 642e6345b9e..326e1e7ae00 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -3,37 +3,32 @@ module API class Labels < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get all labels of the project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/labels + desc 'Get all labels of the project' do + success Entities::Label + end get ':id/labels' do present available_labels, with: Entities::Label, current_user: current_user end - # Creates a new label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be created - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # POST /projects/:id/labels + desc 'Create a new label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be created' + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The description of label to be created' + end post ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name, :color] - - attrs = attributes_for_keys [:name, :color, :description] - label = user_project.find_label(attrs[:name]) + label = user_project.find_label(params[:name]) conflict!('Label already exists') if label - label = user_project.labels.create(attrs) + label = user_project.labels.create(declared(params, include_parent_namespaces: false).to_h) if label.valid? present label, with: Entities::Label, current_user: current_user @@ -42,54 +37,44 @@ module API end end - # Deletes an existing label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # - # Example Request: - # DELETE /projects/:id/labels + desc 'Delete an existing label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end delete ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] label = user_project.find_label(params[:name]) not_found!('Label') unless label - label.destroy + present label.destroy, with: Entities::Label, current_user: current_user end - # Updates an existing label. At least one optional parameter is required. - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # PUT /projects/:id/labels + desc 'Update an existing label. At least one optional parameter is required.' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be updated' + optional :new_name, type: String, desc: 'The new name of the label' + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The new description of label' + at_least_one_of :new_name, :color, :description + end put ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] label = user_project.find_label(params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color, :description] - - if attrs.empty? - render_api_error!('Required parameters "new_name" or "color" ' \ - 'missing', - 400) - end - + update_params = declared(params, + include_parent_namespaces: false, + include_missing: false).to_h # Rename new name to the actual label attribute name - attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) + update_params['name'] = update_params.delete('new_name') if update_params.key?('new_name') - if label.update(attrs) + if label.update(update_params) present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 7b675e05fbb..bf2a199ce21 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -4,25 +4,24 @@ module API before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository tags - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/tags + desc 'Get a project repository tags' do + success Entities::RepoTag + end get ":id/repository/tags" do present user_project.repository.tags.sort_by(&:name).reverse, with: Entities::RepoTag, project: user_project end - # Get a single repository tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # Example Request: - # GET /projects/:id/repository/tags/:tag_name + desc 'Get a single repository tag' do + success Entities::RepoTag + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag @@ -30,20 +29,21 @@ module API present tag, with: Entities::RepoTag, project: user_project end - # Create tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # ref (required) - Create tag from commit sha or branch - # message (optional) - Specifying a message creates an annotated tag. - # Example Request: - # POST /projects/:id/repository/tags + desc 'Create a new repository tag' do + success Entities::RepoTag + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :ref, type: String, desc: 'The commit sha or branch name' + optional :message, type: String, desc: 'Specifying a message creates an annotated tag' + optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database' + end post ':id/repository/tags' do authorize_push_project - message = params[:message] || nil + create_params = declared(params) + result = CreateTagService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], message, params[:release_description]) + execute(create_params[:tag_name], create_params[:ref], create_params[:message], create_params[:release_description]) if result[:status] == :success present result[:tag], @@ -54,15 +54,13 @@ module API end end - # Delete tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # Example Request: - # DELETE /projects/:id/repository/tags/:tag + desc 'Delete a repository tag' + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project + result = DeleteTagService.new(user_project, current_user). execute(params[:tag_name]) @@ -75,17 +73,16 @@ module API end end - # Add release notes to tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # description (required) - Release notes with markdown support - # Example Request: - # POST /projects/:id/repository/tags/:tag_name/release + desc 'Add a release note to a tag' do + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :description, type: String, desc: 'Release notes with markdown support' + end post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - required_attributes! [:description] + result = CreateReleaseService.new(user_project, current_user). execute(params[:tag_name], params[:description]) @@ -96,17 +93,16 @@ module API end end - # Updates a release notes of a tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # description (required) - Release notes with markdown support - # Example Request: - # PUT /projects/:id/repository/tags/:tag_name/release + desc "Update a tag's release note" do + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :description, type: String, desc: 'Release notes with markdown support' + end put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - required_attributes! [:description] + result = UpdateReleaseService.new(user_project, current_user). execute(params[:tag_name], params[:description]) diff --git a/lib/api/users.rb b/lib/api/users.rb index e868f628404..c28e07a76b7 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -333,11 +333,11 @@ module API user = User.find_by(id: declared(params).id) not_found!('User') unless user - events = user.recent_events. + events = user.events. merge(ProjectsFinder.new.execute(current_user)). references(:project). with_associations. - page(params[:page]) + recent present paginate(events), with: Entities::Event end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 9fcd9a3f999..d746070913d 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,11 +2,14 @@ require 'yaml' module Backup class Repository + def dump prepare Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " + path_to_project_repo = path_to_repo(project) + path_to_project_bundle = path_to_bundle(project) # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace @@ -14,8 +17,22 @@ module Backup if project.empty_repo? $progress.puts "[SKIPPED]".color(:cyan) else - cmd = %W(tar -cf #{path_to_bundle(project)} -C #{path_to_repo(project)} .) + in_path(path_to_project_repo) do |dir| + FileUtils.mkdir_p(path_to_tars(project)) + cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + puts "[FAILED]".color(:red) + puts "failed: #{cmd.join(' ')}" + puts output + abort 'Backup failed' + end + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts "[DONE]".color(:green) else @@ -27,19 +44,22 @@ module Backup end wiki = ProjectWiki.new(project) + path_to_wiki_repo = path_to_repo(wiki) + path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_repo(wiki)) + if File.exist?(path_to_wiki_repo) $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? $progress.puts " [SKIPPED]".color(:cyan) else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Backup failed' end end @@ -60,40 +80,59 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " + path_to_project_repo = path_to_repo(project) + path_to_project_bundle = path_to_bundle(project) project.ensure_dir_exist - if File.exist?(path_to_bundle(project)) - FileUtils.mkdir_p(path_to_repo(project)) - cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) + if File.exists?(path_to_project_bundle) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) else - cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)}) + cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) end - if system(*cmd, silent) + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts "[DONE]".color(:green) else puts "[FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Restore failed' end + in_path(path_to_tars(project)) do |dir| + cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + + output, status = Gitlab::Popen.popen(cmd) + unless status.zero? + puts "[FAILED]".color(:red) + puts "failed: #{cmd.join(' ')}" + puts output + abort 'Restore failed' + end + end + wiki = ProjectWiki.new(project) + path_to_wiki_repo = path_to_repo(wiki) + path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_bundle(wiki)) + if File.exist?(path_to_wiki_bundle) $progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then # try to restore with 'git clone --bare'. - FileUtils.rm_rf(path_to_repo(wiki)) - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + FileUtils.rm_rf(path_to_wiki_repo) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo}) - if system(*cmd, silent) + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Restore failed' end end @@ -101,13 +140,15 @@ module Backup $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args - if system(*cmd) + + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd}" + puts output end - end protected @@ -117,11 +158,30 @@ module Backup end def path_to_bundle(project) - File.join(backup_repos_path, project.path_with_namespace + ".bundle") + File.join(backup_repos_path, project.path_with_namespace + '.bundle') + end + + def path_to_tars(project, dir = nil) + path = File.join(backup_repos_path, project.path_with_namespace) + + if dir + File.join(path, "#{dir}.tar") + else + path + end end def backup_repos_path - File.join(Gitlab.config.backup.path, "repositories") + File.join(Gitlab.config.backup.path, 'repositories') + end + + def in_path(path) + return unless Dir.exist?(path) + + dir_entries = Dir.entries(path) + %w[annex custom_hooks].each do |entry| + yield(entry) if dir_entries.include?(entry) + end end def prepare diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index c24831e68ee..9f9a96cdc65 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -39,7 +39,7 @@ module Banzai end def find_labels(project) - LabelsFinder.new(nil, project_id: project.id).execute(authorized_only: false) + LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true) end # Parameters to pass to `Label.find_by` based on the given arguments diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 4fa8d05481f..f09d78be0ce 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -52,8 +52,8 @@ module Banzai relative_url_root, context[:project].path_with_namespace, uri_type(file_path), - ref, - file_path + Addressable::URI.escape(ref), + Addressable::URI.escape(file_path) ].compact.join('/').squeeze('/').chomp('/') uri diff --git a/lib/constraints/namespace_url_constrainer.rb b/lib/constraints/namespace_url_constrainer.rb index 23920193743..91b70143f11 100644 --- a/lib/constraints/namespace_url_constrainer.rb +++ b/lib/constraints/namespace_url_constrainer.rb @@ -1,6 +1,9 @@ class NamespaceUrlConstrainer def matches?(request) - id = request.path.sub(/\A\/+/, '').split('/').first.sub(/.atom\z/, '') + id = request.path + id = id.sub(/\A#{relative_url_root}/, '') if relative_url_root + id = id.sub(/\A\/+/, '').split('/').first + id = id.sub(/.atom\z/, '') if id if id =~ Gitlab::Regex.namespace_regex find_resource(id) @@ -10,4 +13,12 @@ class NamespaceUrlConstrainer def find_resource(id) Namespace.find_by_path(id) end + + private + + def relative_url_root + if defined?(Gitlab::Application.config.relative_url_root) + Gitlab::Application.config.relative_url_root + end + end end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 4f81863da35..d76aa38f741 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -83,7 +83,7 @@ module Gitlab tag = repository.find_tag(tag_name) if tag - commit = repository.commit(tag.target) + commit = repository.commit(tag.dereferenced_target) commit.try(:sha) end else diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb new file mode 100644 index 00000000000..b1a6d5fe0f6 --- /dev/null +++ b/lib/gitlab/ee_compat_check.rb @@ -0,0 +1,261 @@ +# rubocop: disable Rails/Output +module Gitlab + # Checks if a set of migrations requires downtime or not. + class EeCompatCheck + EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + + attr_reader :ce_branch, :check_dir, :ce_repo + + def initialize(branch:, check_dir:, ce_repo: nil) + @ce_branch = branch + @check_dir = check_dir + @ce_repo = ce_repo || 'https://gitlab.com/gitlab-org/gitlab-ce.git' + end + + def check + ensure_ee_repo + delete_patches + + generate_patch(ce_branch, ce_patch_full_path) + + Dir.chdir(check_dir) do + step("In the #{check_dir} directory") + + step("Pulling latest master", %w[git pull --ff-only origin master]) + + status = catch(:halt_check) do + ce_branch_compat_check! + + delete_ee_branch_locally + + ee_branch_presence_check! + + ee_branch_compat_check! + end + + delete_ee_branch_locally + delete_patches + + if status.nil? + true + else + false + end + end + end + + private + + def ensure_ee_repo + if Dir.exist?(check_dir) + step("#{check_dir} already exists") + else + cmd = %W[git clone --branch master --single-branch --depth 1 #{EE_REPO} #{check_dir}] + step("Cloning #{EE_REPO} into #{check_dir}", cmd) + end + end + + def ce_branch_compat_check! + cmd = %W[git apply --check #{ce_patch_full_path}] + status = step("Checking if #{ce_patch_name} applies cleanly to EE/master", cmd) + + if status.zero? + puts ce_applies_cleanly_msg(ce_branch) + throw(:halt_check) + end + end + + def ee_branch_presence_check! + status = step("Fetching origin/#{ee_branch}", %W[git fetch origin #{ee_branch}]) + + unless status.zero? + puts + puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + + throw(:halt_check, :ko) + end + end + + def ee_branch_compat_check! + step("Checking out origin/#{ee_branch}", %W[git checkout -b #{ee_branch} FETCH_HEAD]) + + generate_patch(ee_branch, ee_patch_full_path) + cmd = %W[git apply --check #{ee_patch_full_path}] + status = step("Checking if #{ee_patch_name} applies cleanly to EE/master", cmd) + + unless status.zero? + puts + puts ee_branch_doesnt_apply_cleanly_msg + + throw(:halt_check, :ko) + end + + puts + puts ee_applies_cleanly_msg + end + + def generate_patch(branch, filepath) + FileUtils.rm(filepath, force: true) + + depth = 0 + loop do + depth += 10 + step("Fetching origin/master", %W[git fetch origin master --depth=#{depth}]) + status = step("Finding merge base with master", %W[git merge-base FETCH_HEAD #{branch}]) + + break if status.zero? || depth > 500 + end + + raise "#{branch} is too far behind master, please rebase it!" if depth > 500 + + step("Generating the patch against master") + output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) + throw(:halt_check, :ko) unless status.zero? + + File.write(filepath, output) + throw(:halt_check, :ko) unless File.exist?(filepath) + end + + def delete_ee_branch_locally + command(%w[git checkout master]) + step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}]) + end + + def delete_patches + step("Deleting #{ce_patch_full_path}") + FileUtils.rm(ce_patch_full_path, force: true) + + step("Deleting #{ee_patch_full_path}") + FileUtils.rm(ee_patch_full_path, force: true) + end + + def ce_patch_name + @ce_patch_name ||= "#{ce_branch}.patch" + end + + def ce_patch_full_path + @ce_patch_full_path ||= File.expand_path(ce_patch_name, check_dir) + end + + def ee_branch + @ee_branch ||= "#{ce_branch}-ee" + end + + def ee_patch_name + @ee_patch_name ||= "#{ee_branch}.patch" + end + + def ee_patch_full_path + @ee_patch_full_path ||= File.expand_path(ee_patch_name, check_dir) + end + + def step(desc, cmd = nil) + puts "\n=> #{desc}\n" + + if cmd + puts "\n$ #{cmd.join(' ')}" + command(cmd) + end + end + + def command(cmd) + output, status = Gitlab::Popen.popen(cmd) + puts output + + status + end + + def ce_applies_cleanly_msg(ce_branch) + <<-MSG.strip_heredoc + ================================================================= + 🎉 Congratulations!! 🎉 + + The #{ce_branch} branch applies cleanly to EE/master! + + Much ❤️!! + =================================================================\n + MSG + end + + def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} branch does not apply cleanly to the current + EE/master, and no #{ee_branch} branch was found in the EE repository. + + Please create a #{ee_branch} branch that includes changes from + #{ce_branch} but also specific changes than can be applied cleanly + to EE/master. + + There are different ways to create such branch: + + 1. Create a new branch based on the CE branch and rebase it on top of EE/master + + # In the EE repo + $ git fetch #{ce_repo} #{ce_branch} + $ git checkout -b #{ee_branch} FETCH_HEAD + + # You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit + # before rebasing to limit the conflicts-resolving steps during the rebase + $ git fetch origin + $ git rebase origin/master + + At this point you will likely have conflicts. + Solve them, and continue/finish the rebase. + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE". + + 2. Create a new branch from master and cherry-pick your CE commits + + # In the EE repo + $ git fetch origin + $ git checkout -b #{ee_branch} FETCH_HEAD + $ git fetch #{ce_repo} #{ce_branch} + $ git cherry-pick SHA # Repeat for all the commits you want to pick + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit. + + Don't forget to push your branch to #{EE_REPO}: + + # In the EE repo + $ git push origin #{ee_branch} + + You can then retry this failed build, and hopefully it should pass. + + Stay 💪 ! + =================================================================\n + MSG + end + + def ee_branch_doesnt_apply_cleanly_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} does not apply cleanly to the current + EE/master, and even though a #{ee_branch} branch exists in the EE + repository, it does not apply cleanly either to EE/master! + + Please update the #{ee_branch}, push it again to #{EE_REPO}, and + retry this build. + + Stay 💪 ! + =================================================================\n + MSG + end + + def ee_applies_cleanly_msg + <<-MSG.strip_heredoc + ================================================================= + 🎉 Congratulations!! 🎉 + + The #{ee_branch} branch applies cleanly to EE/master! + + Much ❤️!! + =================================================================\n + MSG + end + end +end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index ffe49364379..7e8f35e9298 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -27,7 +27,7 @@ module Gitlab # on begin/ensure blocks to cancel a lease, because the 'ensure' does # not always run. Think of 'kill -9' from the Unicorn master for # instance. - # + # # If you find that leases are getting in your way, ask yourself: would # it be enough to lower the lease timeout? Another thing that might be # appropriate is to only use a lease for bulk/automated operations, and @@ -48,6 +48,13 @@ module Gitlab end end + # Returns true if the key for this lease is set. + def exists? + Gitlab::Redis.with do |redis| + redis.exists(redis_key) + end + end + # No #cancel method. See comments above! private diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 65ee85ca5a9..222bcdcbf9c 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -75,7 +75,7 @@ module Gitlab def create_label(name) params = { title: name, color: nice_label_color(name) } - ::Labels::FindOrCreateService.new(project.owner, project, params).execute + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end def user_info(person_id) @@ -133,7 +133,7 @@ module Gitlab updated_at: DateTime.parse(bug['dtLastUpdated']) ) - issue_labels = ::LabelsFinder.new(project.owner, project_id: project.id, title: labels).execute + issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 8cacf4f4925..6dbae64a9fe 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -10,7 +10,9 @@ module Gitlab end def create! - self.klass.create!(self.attributes) + project.public_send(project_association).find_or_create_by!(find_condition) do |record| + record.attributes = attributes + end end private diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 7f424b74efb..85df6547a67 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -105,18 +105,20 @@ module Gitlab data = api.send(method, *args) return data unless data.is_a?(Array) + last_response = api.last_response + if block_given? yield data - each_response_page(&block) + # api.last_response could change while we're yielding (e.g. fetching labels for each PR) + # so we cache our own last response + each_response_page(last_response, &block) else - each_response_page { |page| data.concat(page) } + each_response_page(last_response) { |page| data.concat(page) } data end end - def each_response_page - last_response = api.last_response - + def each_response_page(last_response) while last_response.rels[:next] sleep rate_limit_sleep_time if rate_limit_exceed? last_response = last_response.rels[:next].get diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 4b70f33a851..ecc28799737 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -24,7 +24,8 @@ module Gitlab import_milestones import_issues import_pull_requests - import_comments + import_comments(:issues) + import_comments(:pull_requests) import_wiki import_releases handle_errors @@ -48,7 +49,7 @@ module Gitlab end def import_labels - client.labels(repo, per_page: 100) do |labels| + fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| begin label = LabelFormatter.new(project, raw).create! @@ -61,7 +62,7 @@ module Gitlab end def import_milestones - client.milestones(repo, state: :all, per_page: 100) do |milestones| + fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| begin MilestoneFormatter.new(project, raw).create! @@ -73,7 +74,7 @@ module Gitlab end def import_issues - client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| + fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| gh_issue = IssueFormatter.new(project, raw) @@ -90,7 +91,7 @@ module Gitlab end def import_pull_requests - client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| + fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| pull_request = PullRequestFormatter.new(project, raw) next unless pull_request.valid? @@ -132,8 +133,15 @@ module Gitlab end def apply_labels(issuable, raw_issuable) - if raw_issuable.labels.count > 0 - label_ids = raw_issuable.labels + # GH returns labels for issues but not for pull requests! + labels = if issuable.is_a?(MergeRequest) + client.labels_for_issue(repo, raw_issuable.number) + else + raw_issuable.labels + end + + if labels.count > 0 + label_ids = labels .map { |attrs| @labels[attrs.name] } .compact @@ -141,23 +149,35 @@ module Gitlab end end - def import_comments - client.issues_comments(repo, per_page: 100) do |comments| - create_comments(comments, :issue) - end + def import_comments(issuable_type) + resource_type = "#{issuable_type}_comments".to_sym + + # Two notes here: + # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note, + # compare it against every comment in the current imported page until we find match, and that's where start importing + # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns + # only comments on diffs, so select last note not based on noteable_type but on line_code + line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL' + last_note = project.notes.where("line_code IS #{line_code_is}").last + + fetch_resources(resource_type, repo, per_page: 100) do |comments| + if last_note + discard_inserted_comments(comments, last_note) + last_note = nil + end - client.pull_requests_comments(repo, per_page: 100) do |comments| - create_comments(comments, :pull_request) + create_comments(comments) end end - def create_comments(comments, issuable_type) + def create_comments(comments) ActiveRecord::Base.no_touching do comments.each do |raw| begin - comment = CommentFormatter.new(project, raw) - issuable_class = issuable_type == :issue ? Issue : MergeRequest - iid = raw.send("#{issuable_type}_url").split('/').last # GH doesn't return parent ID directly + comment = CommentFormatter.new(project, raw) + # GH does not return info about comment's parent, so we guess it by checking its URL! + *_, parent, iid = URI(raw.html_url).path.split('/') + issuable_class = parent == 'issues' ? Issue : MergeRequest issuable = issuable_class.find_by_iid(iid) next unless issuable @@ -169,6 +189,24 @@ module Gitlab end end + def discard_inserted_comments(comments, last_note) + last_note_attrs = nil + + cut_off_index = comments.find_index do |raw| + comment = CommentFormatter.new(project, raw) + comment_attrs = comment.attributes + last_note_attrs ||= last_note.slice(*comment_attrs.keys) + + comment_attrs.with_indifferent_access == last_note_attrs + end + + # No matching resource in the collection, which means we got halted right on the end of the last page, so all good + return unless cut_off_index + + # Otherwise, remove the resources we've already inserted + comments.shift(cut_off_index + 1) + end + def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) @@ -184,7 +222,7 @@ module Gitlab end def import_releases - client.releases(repo, per_page: 100) do |releases| + fetch_resources(:releases, repo, per_page: 100) do |releases| releases.each do |raw| begin gh_release = ReleaseFormatter.new(project, raw) @@ -195,6 +233,47 @@ module Gitlab end end end + + def fetch_resources(resource_type, *opts) + return if imported?(resource_type) + + opts.last.merge!(page: current_page(resource_type)) + + client.public_send(resource_type, *opts) do |resources| + yield resources + increment_page(resource_type) + end + + imported!(resource_type) + end + + def imported?(resource_type) + Rails.cache.read("#{cache_key_prefix}:#{resource_type}:imported") + end + + def imported!(resource_type) + Rails.cache.write("#{cache_key_prefix}:#{resource_type}:imported", true, ex: 1.day) + end + + def increment_page(resource_type) + key = "#{cache_key_prefix}:#{resource_type}:current-page" + + # Rails.cache.increment calls INCRBY directly on the value stored under the key, which is + # a serialized ActiveSupport::Cache::Entry, so it will return an error by Redis, hence this ugly work-around + page = Rails.cache.read(key) + page += 1 + Rails.cache.write(key, page) + + page + end + + def current_page(resource_type) + Rails.cache.fetch("#{cache_key_prefix}:#{resource_type}:current-page", ex: 1.day) { 1 } + end + + def cache_key_prefix + @cache_key_prefix ||= "github-import:#{project.id}" + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 77621de9f4c..8c32ac59fc5 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -20,8 +20,12 @@ module Gitlab raw_data.comments > 0 end - def klass - Issue + def project_association + :issues + end + + def find_condition + { iid: number } end def number diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index 942dfb3312b..211ccdc51bb 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -9,14 +9,14 @@ module Gitlab } end - def klass - Label + def project_association + :labels end def create! params = attributes.except(:project) - service = ::Labels::FindOrCreateService.new(project.owner, project, params) - label = service.execute + service = ::Labels::FindOrCreateService.new(nil, project, params) + label = service.execute(skip_authorization: true) raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index b2fa524cf5b..401dd962521 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -14,8 +14,12 @@ module Gitlab } end - def klass - Milestone + def project_association + :milestones + end + + def find_condition + { iid: raw_data.number } end private diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 1408683100f..b9a227fb11a 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -24,8 +24,12 @@ module Gitlab } end - def klass - MergeRequest + def project_association + :merge_requests + end + + def find_condition + { iid: number } end def number diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb index 73d643b00ad..1ad702a6058 100644 --- a/lib/gitlab/github_import/release_formatter.rb +++ b/lib/gitlab/github_import/release_formatter.rb @@ -11,8 +11,12 @@ module Gitlab } end - def klass - Release + def project_association + :releases + end + + def find_condition + { tag: raw_data.tag_name } end def valid? diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 6a68e786b4f..1f4edc36928 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -101,7 +101,7 @@ module Gitlab state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) - issue_labels = ::LabelsFinder.new(project.owner, project_id: project.id, title: labels).execute + issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) @@ -235,7 +235,7 @@ module Gitlab def create_label(name) params = { name: name, color: nice_label_color(name) } - ::Labels::FindOrCreateService.new(project.owner, project, params).execute + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end def format_content(raw_content) diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb index 01a2c19ab23..b8ca7f2f55f 100644 --- a/lib/gitlab/issues_labels.rb +++ b/lib/gitlab/issues_labels.rb @@ -19,7 +19,7 @@ module Gitlab ] labels.each do |params| - ::Labels::FindOrCreateService.new(project.owner, project).execute(params) + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 12999a90a29..a5220d92312 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -33,7 +33,12 @@ module Gitlab config[:mailbox] = 'inbox' if config[:mailbox].nil? if config[:enabled] && config[:address] - config[:redis_url] = Gitlab::Redis.new(rails_env).url + gitlab_redis = Gitlab::Redis.new(rails_env) + config[:redis_url] = gitlab_redis.url + + if gitlab_redis.sentinels? + config[:sentinels] = gitlab_redis.sentinels + end end config diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb new file mode 100644 index 00000000000..879d46446b3 --- /dev/null +++ b/lib/gitlab/optimistic_locking.rb @@ -0,0 +1,19 @@ +module Gitlab + module OptimisticLocking + extend self + + def retry_lock(subject, retries = 100, &block) + loop do + begin + ActiveRecord::Base.transaction do + return block.call(subject) + end + rescue ActiveRecord::StaleObjectError + retries -= 1 + raise unless retries >= 0 + subject.reload + end + end + end + end +end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index c649da8c426..9226da2d6b1 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -63,6 +63,14 @@ module Gitlab raw_config_hash[:url] end + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + private def redis_store_options diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index a95a3455a4a..78ae187817a 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -29,5 +29,5 @@ namespace :cache do task all: [:db, :redis] end - task clear: 'cache:clear:all' + task clear: 'cache:clear:redis' end diff --git a/lib/tasks/ce_to_ee_merge_check.rake b/lib/tasks/ce_to_ee_merge_check.rake deleted file mode 100644 index 424e7883060..00000000000 --- a/lib/tasks/ce_to_ee_merge_check.rake +++ /dev/null @@ -1,4 +0,0 @@ -desc 'Checks if the branch would apply cleanly to EE' -task ce_to_ee_merge_check: :environment do - Rake::Task['gitlab:dev:ce_to_ee_merge_check'].invoke -end diff --git a/lib/tasks/ee_compat_check.rake b/lib/tasks/ee_compat_check.rake new file mode 100644 index 00000000000..f494fa5c5c2 --- /dev/null +++ b/lib/tasks/ee_compat_check.rake @@ -0,0 +1,4 @@ +desc 'Checks if the branch would apply cleanly to EE' +task ee_compat_check: :environment do + Rake::Task['gitlab:dev:ee_compat_check'].invoke +end diff --git a/lib/tasks/eslint.rake b/lib/tasks/eslint.rake new file mode 100644 index 00000000000..d43cbad1909 --- /dev/null +++ b/lib/tasks/eslint.rake @@ -0,0 +1,7 @@ +unless Rails.env.production? + desc "GitLab | Run ESLint" + task :eslint do + system("npm", "run", "eslint") + end +end + diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index b43ee5b3383..a9f1255e8cf 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -51,6 +51,7 @@ namespace :gitlab do $progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') @@ -58,6 +59,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke backup.cleanup end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 47bdb2d32d2..5ee99dfc810 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -1,106 +1,21 @@ namespace :gitlab do namespace :dev do desc 'Checks if the branch would apply cleanly to EE' - task ce_to_ee_merge_check: :environment do + task ee_compat_check: :environment do return if defined?(Gitlab::License) return unless ENV['CI'] - ce_repo = ENV['CI_BUILD_REPO'] - ce_branch = ENV['CI_BUILD_REF_NAME'] - - ee_repo = 'https://gitlab.com/gitlab-org/gitlab-ee.git' - ee_branch = "#{ce_branch}-ee" - ee_dir = 'gitlab-ee-merge-check' - - puts "\n=> Cloning #{ee_repo} into #{ee_dir}\n" - `git clone #{ee_repo} #{ee_dir} --depth 1` - Dir.chdir(ee_dir) do - puts "\n => Fetching #{ce_repo}/#{ce_branch}\n" - `git fetch #{ce_repo} #{ce_branch} --depth 1` - - # Try to merge the current tested branch to EE/master... - puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" - `git merge FETCH_HEAD` - - exit 0 if $?.success? - - # Check if the <branch>-ee branch exists... - puts "\n => Check if #{ee_repo}/#{ee_branch} exists\n" - `git rev-parse --verify #{ee_branch}` - - # The <branch>-ee doesn't exist - unless $?.success? - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to the - current EE/master, and no #{ee_branch} branch was detected in - the EE repository. - - Please create a #{ee_branch} branch that includes changes from - #{ce_branch} but also specific changes than can be applied cleanly - to EE/master. - - You can create this branch as follows: - - 1. In the EE repo: - $ git fetch origin - $ git fetch #{ce_repo} #{ce_branch} - $ git checkout -b #{ee_branch} FETCH_HEAD - $ git rebase origin/master - 2. At this point you will likely have conflicts, solve them, and - continue/finish the rebase. Note: You can squash the CE commits - before rebasing. - 3. You can squash all the original #{ce_branch} commits into a - single "Port of #{ce_branch} to EE". - 4. Push your branch to #{ee_repo}: - $ git push origin #{ee_branch} - =================================================================\n - MSG - - exit 1 - end - - # Try to merge the <branch>-ee branch to EE/master... - puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n" - `git merge #{ee_branch} master` - - # The <branch>-ee cannot be merged cleanly to EE/master... - unless $?.success? - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to - EE/master, and even though the #{ee_branch} branch exists in the EE - repository, it cannot be merged without conflicts to EE/master. - - Please update the #{ee_branch}, push it again to #{ee_repo}, and - retry this job. - =================================================================\n - MSG - - exit 2 - end - - puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" - `git merge FETCH_HEAD` - exit 0 if $?.success? - - # The <branch>-ee can be merged cleanly to EE/master, but <branch> still - # cannot be merged cleanly to EE/master... - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to EE, and - even though the #{ee_branch} branch exists in the EE repository and - applies cleanly to EE/master, it doesn't prevent conflicts when - merging #{ce_branch} into EE. - - We may be in a complex situation here. - =================================================================\n - MSG - - exit 3 + success = + Gitlab::EeCompatCheck.new( + branch: ENV['CI_BUILD_REF_NAME'], + check_dir: File.expand_path('ee-compat-check', __dir__), + ce_repo: ENV['CI_BUILD_REPO'] + ).check + + if success + exit 0 + else + exit 1 end end end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake new file mode 100644 index 00000000000..32b668df3bf --- /dev/null +++ b/lib/tasks/lint.rake @@ -0,0 +1,9 @@ +unless Rails.env.production? + namespace :lint do + desc "GitLab | lint | Lint JavaScript files using ESLint" + task :javascript do + Rake::Task['eslint'].invoke + end + end +end + diff --git a/lib/tasks/teaspoon.rake b/lib/tasks/teaspoon.rake new file mode 100644 index 00000000000..08caedd7ff3 --- /dev/null +++ b/lib/tasks/teaspoon.rake @@ -0,0 +1,25 @@ +unless Rails.env.production? + Rake::Task['teaspoon'].clear if Rake::Task.task_defined?('teaspoon') + + namespace :teaspoon do + desc 'GitLab | Teaspoon | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures) do |t| + ENV['NO_KNAPSACK'] = 'true' + t.pattern = 'spec/javascripts/fixtures/*.rb' + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Teaspoon | Run JavaScript tests' + task :tests do + require "teaspoon/console" + options = {} + abort('rake teaspoon:tests failed') if Teaspoon::Console.new(options).failures? + end + end + + desc 'GitLab | Teaspoon | Shortcut for teaspoon:fixtures and teaspoon:tests' + task :teaspoon do + Rake::Task['teaspoon:fixtures'].invoke + Rake::Task['teaspoon:tests'].invoke + end +end |