diff options
| author | Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> | 2015-02-27 13:01:57 -0800 |
|---|---|---|
| committer | Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> | 2015-02-27 13:01:57 -0800 |
| commit | 0d22b75b03496ced3d783f8fee9584098602ea1c (patch) | |
| tree | c7ddec6072c716fd63a8703f2dfeb0e4234a633f /lib | |
| parent | 5f682094d9b7c985ad62ebe29664bb6fe87b54be (diff) | |
| parent | d4aab6528cb80b0f41bdac2240dd9cc32543481d (diff) | |
| download | gitlab-ce-0d22b75b03496ced3d783f8fee9584098602ea1c.tar.gz | |
Merge branch 'master' into mmonaco/gitlab-ce-api-user-noconfirm
Conflicts:
lib/api/users.rb
Diffstat (limited to 'lib')
92 files changed, 2224 insertions, 422 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index d26667ba3f7..60858a39407 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -2,10 +2,11 @@ Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file} module API class API < Grape::API + include APIGuard version 'v3', using: :path rescue_from ActiveRecord::RecordNotFound do - rack_response({'message' => '404 Not found'}.to_json, 404) + rack_response({ 'message' => '404 Not found' }.to_json, 404) end rescue_from :all do |exception| @@ -18,7 +19,7 @@ module API message << " " << trace.join("\n ") API.logger.add Logger::FATAL, message - rack_response({'message' => '500 Internal Server Error'}, 500) + rack_response({ 'message' => '500 Internal Server Error' }, 500) end format :json diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb new file mode 100644 index 00000000000..b9994fcefda --- /dev/null +++ b/lib/api/api_guard.rb @@ -0,0 +1,172 @@ +# Guard API with OAuth 2.0 Access Token + +require 'rack/oauth2' + +module APIGuard + extend ActiveSupport::Concern + + included do |base| + # OAuth2 Resource Server Authentication + use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| + # The authenticator only fetches the raw token string + + # Must yield access token to store it in the env + request.access_token + end + + helpers HelperMethods + + install_error_responders(base) + end + + # Helper Methods for Grape Endpoint + module HelperMethods + # Invokes the doorkeeper guard. + # + # If token is presented and valid, then it sets @current_user. + # + # If the token does not have sufficient scopes to cover the requred scopes, + # then it raises InsufficientScopeError. + # + # If the token is expired, then it raises ExpiredError. + # + # If the token is revoked, then it raises RevokedError. + # + # If the token is not found (nil), then it raises TokenNotFoundError. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def doorkeeper_guard!(scopes: []) + if (access_token = find_access_token).nil? + raise TokenNotFoundError + + else + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end + end + end + + def doorkeeper_guard(scopes: []) + if access_token = find_access_token + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError + + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError + + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end + end + end + + def current_user + @current_user + end + + private + def find_access_token + @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + end + + def doorkeeper_request + @doorkeeper_request ||= ActionDispatch::Request.new(env) + end + + def validate_access_token(access_token, scopes) + Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) + end + end + + module ClassMethods + # Installs the doorkeeper guard on the whole Grape API endpoint. + # + # Arguments: + # + # scopes: (optional) scopes required for this guard. + # Defaults to empty array. + # + def guard_all!(scopes: []) + before do + guard! scopes: scopes + end + end + + private + def install_error_responders(base) + error_classes = [ MissingTokenError, TokenNotFoundError, + ExpiredError, RevokedError, InsufficientScopeError] + + base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler + end + + def oauth2_bearer_token_error_handler + Proc.new do |e| + response = + case e + when MissingTokenError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new + + when TokenNotFoundError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Bad Access Token.") + + when ExpiredError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token is expired. You can either do re-authorization or token refresh.") + + when RevokedError + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token was revoked. You have to re-authorize from the user.") + + when InsufficientScopeError + # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) + # does not include WWW-Authenticate header, which breaks the standard. + Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( + :insufficient_scope, + Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope], + { scope: e.scopes }) + end + + response.finish + end + end + end + + # + # Exceptions + # + + class MissingTokenError < StandardError; end + + class TokenNotFoundError < StandardError; end + + class ExpiredError < StandardError; end + + class RevokedError < StandardError; end + + class InsufficientScopeError < StandardError + attr_reader :scopes + def initialize(scopes) + @scopes = scopes + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 6ec1a753a69..b52d786e020 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -14,7 +14,8 @@ module API # Example Request: # GET /projects/:id/repository/branches get ":id/repository/branches" do - present user_project.repository.branches.sort_by(&:name), with: Entities::RepoObject, project: user_project + branches = user_project.repository.branches.sort_by(&:name) + present branches, with: Entities::RepoObject, project: user_project end # Get a single branch @@ -26,7 +27,7 @@ module API # GET /projects/:id/repository/branches/:branch get ':id/repository/branches/:branch', requirements: { branch: /.*/ } do @branch = user_project.repository.branches.find { |item| item.name == params[:branch] } - not_found!("Branch does not exist") if @branch.nil? + not_found!("Branch") unless @branch present @branch, with: Entities::RepoObject, project: user_project end @@ -43,7 +44,7 @@ module API authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) - not_found! unless @branch + not_found!("Branch") unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) user_project.protected_branches.create(name: @branch.name) unless protected_branch @@ -63,7 +64,7 @@ module API authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) - not_found! unless @branch + not_found!("Branch does not exist") unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) protected_branch.destroy if protected_branch diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 4a67313430a..0de4e720ffe 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -50,6 +50,67 @@ module API not_found! "Commit" unless commit commit.diffs end + + # Get a commit's comments + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The commit hash + # Examples: + # GET /projects/:id/repository/commits/:sha/comments + get ':id/repository/commits/:sha/comments' do + sha = params[:sha] + commit = user_project.repository.commit(sha) + not_found! 'Commit' unless commit + notes = Note.where(commit_id: commit.id) + present paginate(notes), with: Entities::CommitNote + end + + # Post comment to commit + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The commit hash + # note (required) - Text of comment + # path (optional) - The file path + # line (optional) - The line number + # line_type (optional) - The type of line (new or old) + # Examples: + # POST /projects/:id/repository/commits/:sha/comments + post ':id/repository/commits/:sha/comments' do + required_attributes! [:note] + + sha = params[:sha] + commit = user_project.repository.commit(sha) + not_found! 'Commit' unless commit + opts = { + note: params[:note], + noteable_type: 'Commit', + commit_id: commit.id + } + + if params[:path] && params[:line] && params[:line_type] + commit.diffs.each do |diff| + next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.lines.to_a) + + lines.each do |line| + next unless line.new_pos == params[:line].to_i && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + end + + break if opts[:line_code] + end + end + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: Entities::CommitNote + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4e7b1c91c4e..7572104fc16 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -14,9 +14,14 @@ module API expose :bio, :skype, :linkedin, :twitter, :website_url end + class Identity < Grape::Entity + expose :provider, :extern_uid + end + class UserFull < User expose :email - expose :theme_id, :color_scheme_id, :extern_uid, :provider + expose :theme_id, :color_scheme_id, :projects_limit + expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project end @@ -50,7 +55,7 @@ module API expose :path, :path_with_namespace expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :snippets_enabled, :created_at, :last_activity_at expose :namespace - expose :forked_from_project, using: Entities::ForkedFromProject, :if => lambda{ | project, options | project.forked? } + expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ | project, options | project.forked? } end class ProjectMember < UserBasic @@ -60,7 +65,7 @@ module API end class Group < Grape::Entity - expose :id, :name, :path, :owner_id + expose :id, :name, :path, :description end class GroupDetail < Group @@ -142,6 +147,11 @@ module API expose :state, :created_at, :updated_at end + class RepoDiff < Grape::Entity + expose :old_path, :new_path, :a_mode, :b_mode, :diff + expose :new_file, :renamed_file, :deleted_file + end + class Milestone < ProjectEntity expose :due_date end @@ -161,6 +171,12 @@ module API expose :milestone, using: Entities::Milestone end + class MergeRequestChanges < MergeRequest + expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| + compare.diffs + end + end + class SSHKey < Grape::Entity expose :id, :title, :key, :created_at end @@ -178,6 +194,14 @@ module API expose :author, using: Entities::UserBasic end + class CommitNote < Grape::Entity + expose :note + expose(:path) { |note| note.diff_file_name } + expose(:line) { |note| note.diff_new_line } + expose(:line_type) { |note| note.diff_line_type } + expose :author, using: Entities::UserBasic + end + class Event < Grape::Entity expose :title, :project_id, :action_name expose :target_id, :target_type, :author_id @@ -223,11 +247,6 @@ module API expose :name, :color end - class RepoDiff < Grape::Entity - expose :old_path, :new_path, :a_mode, :b_mode, :diff - expose :new_file, :renamed_file, :deleted_file - end - class Compare < Grape::Entity expose :commit, using: Entities::RepoCommit do |compare, options| Commit.decorate(compare.commits).last @@ -251,5 +270,9 @@ module API class Contributor < Grape::Entity expose :name, :email, :commits, :additions, :deletions end + + class BroadcastMessage < Grape::Entity + expose :message, :starts_at, :ends_at, :color, :font + end end end diff --git a/lib/api/files.rb b/lib/api/files.rb index 84e1d311781..3176ef0e256 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -35,7 +35,7 @@ module API file_path = attrs.delete(:file_path) commit = user_project.repository.commit(ref) - not_found! "Commit" unless commit + not_found! 'Commit' unless commit blob = user_project.repository.blob_at(commit.sha, file_path) @@ -53,7 +53,7 @@ module API commit_id: commit.id, } else - render_api_error!('File not found', 404) + not_found! 'File' end end @@ -117,7 +117,8 @@ module API branch_name: branch_name } else - render_api_error!(result[:message], 400) + http_status = result[:http_status] || 400 + render_api_error!(result[:message], http_status) end end diff --git a/lib/api/group_members.rb b/lib/api/group_members.rb index d596517c816..c9c9ccbcb2e 100644 --- a/lib/api/group_members.rb +++ b/lib/api/group_members.rb @@ -3,22 +3,6 @@ module API before { authenticate! } resource :groups do - helpers do - def find_group(id) - group = Group.find(id) - - if can?(current_user, :read_group, group) - group - else - render_api_error!("403 Forbidden - #{current_user.username} lacks sufficient access to #{group.name}", 403) - end - end - - def validate_access_level?(level) - Gitlab::Access.options_with_owner.values.include? level.to_i - end - end - # Get a list of group members viewable by the authenticated user. # # Example Request: @@ -56,6 +40,30 @@ module API present member.user, with: Entities::GroupMember, group: group end + # Update group member + # + # Parameters: + # id (required) - The ID of a group + # user_id (required) - The ID of a group member + # access_level (required) - Project access level + # Example Request: + # PUT /groups/:id/members/:user_id + put ':id/members/:user_id' do + group = find_group(params[:id]) + authorize! :manage_group, group + required_attributes! [:access_level] + + team_member = group.group_members.find_by(user_id: params[:user_id]) + not_found!('User can not be found') if team_member.nil? + + if team_member.update_attributes(access_level: params[:access_level]) + @member = team_member.user + present @member, with: Entities::GroupMember, group: group + else + handle_member_errors team_member.errors + end + end + # Remove member. # # Parameters: diff --git a/lib/api/groups.rb b/lib/api/groups.rb index f0ab6938b1c..a92abd4b690 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -4,32 +4,19 @@ module API before { authenticate! } resource :groups do - helpers do - def find_group(id) - group = Group.find(id) - - if can?(current_user, :read_group, group) - group - else - render_api_error!("403 Forbidden - #{current_user.username} lacks sufficient access to #{group.name}", 403) - end - end - - def validate_access_level?(level) - Gitlab::Access.options_with_owner.values.include? level.to_i - end - end - # Get a groups list # # Example Request: # GET /groups get do - if current_user.admin - @groups = paginate Group - else - @groups = paginate current_user.groups - end + @groups = if current_user.admin + Group.all + else + current_user.groups + end + + @groups = @groups.search(params[:search]) if params[:search].present? + @groups = paginate @groups present @groups, with: Entities::Group end @@ -44,14 +31,14 @@ module API authenticated_as_admin! required_attributes! [:name, :path] - attrs = attributes_for_keys [:name, :path] + attrs = attributes_for_keys [:name, :path, :description] @group = Group.new(attrs) - @group.owner = current_user if @group.save + @group.add_owner(current_user) present @group, with: Entities::Group else - not_found! + render_api_error!("Failed to save group #{@group.errors.messages}", 400) end end @@ -94,7 +81,7 @@ module API if result present group else - not_found! + render_api_error!("Failed to transfer project #{project.errors.messages}", 400) end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 027fb20ec46..228a719fbdf 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -11,7 +11,7 @@ module API def current_user private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - @current_user ||= User.find_by(authentication_token: private_token) + @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard) unless @current_user && Gitlab::UserAccess.allowed?(@current_user) return nil @@ -42,7 +42,7 @@ module API def user_project @project ||= find_project(params[:id]) - @project || not_found! + @project || not_found!("Project") end def find_project(id) @@ -55,6 +55,21 @@ module API end end + def find_group(id) + begin + group = Group.find(id) + rescue ActiveRecord::RecordNotFound + group = Group.find_by!(path: id) + end + + if can?(current_user, :read_group, group) + group + else + forbidden!("#{current_user.username} lacks sufficient "\ + "access to #{group.name}") + end + end + def paginate(relation) per_page = params[:per_page].to_i paginated = relation.page(params[:page]).per(per_page) @@ -68,7 +83,7 @@ module API end def authenticate_by_gitlab_shell_token! - unauthorized! unless secret_token == params['secret_token'] + unauthorized! unless secret_token == params['secret_token'].try(:chomp) end def authenticated_as_admin! @@ -135,10 +150,32 @@ module API errors end + def validate_access_level?(level) + Gitlab::Access.options_with_owner.values.include? level.to_i + end + + def issuable_order_by + if params["order_by"] == 'updated_at' + 'updated_at' + else + 'created_at' + end + end + + def issuable_sort + if params["sort"] == 'asc' + :asc + else + :desc + end + end + # error helpers - def forbidden! - render_api_error!('403 Forbidden', 403) + def forbidden!(reason = nil) + message = ['403 Forbidden'] + message << " - #{reason}" if reason + render_api_error!(message.join(' '), 403) end def bad_request!(attribute) @@ -173,7 +210,7 @@ module API end def render_api_error!(message, status) - error!({'message' => message}, status) + error!({ 'message' => message }, status) end private @@ -199,7 +236,12 @@ module API end def secret_token - File.read(Rails.root.join('.gitlab_shell_secret')) + File.read(Rails.root.join('.gitlab_shell_secret')).chomp + end + + def handle_member_errors(errors) + error!(errors[:access_level], 422) if errors[:access_level].any? + not_found!(errors) end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ebf2296097d..ba3fe619b92 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -1,9 +1,7 @@ module API # Internal access API class Internal < Grape::API - before { - authenticate_by_gitlab_shell_token! - } + before { authenticate_by_gitlab_shell_token! } namespace 'internal' do # Check if git command is allowed to project @@ -25,25 +23,30 @@ module API # project. This applies the correct project permissions to # the wiki repository as well. access = - if project_path =~ /\.wiki\Z/ - project_path.sub!(/\.wiki\Z/, '') + if project_path.end_with?('.wiki') + project_path.chomp!('.wiki') Gitlab::GitAccessWiki.new else Gitlab::GitAccess.new end project = Project.find_with_namespace(project_path) - return false unless project + + unless project + return Gitlab::GitAccessStatus.new(false, 'No such project') + end actor = if params[:key_id] - Key.find(params[:key_id]) + Key.find_by(id: params[:key_id]) elsif params[:user_id] - User.find(params[:user_id]) + User.find_by(id: params[:user_id]) end - return false unless actor + unless actor + return Gitlab::GitAccessStatus.new(false, 'No such user or key') + end - access.allowed?( + access.check( actor, params[:action], project, @@ -66,6 +69,14 @@ module API gitlab_rev: Gitlab::REVISION, } end + + get "/broadcast_message" do + if message = BroadcastMessage.current + present message, with: Entities::BroadcastMessage + else + {} + end + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index d2828b24c36..ff062be6040 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -27,7 +27,9 @@ module API # Parameters: # state (optional) - Return "opened" or "closed" issues # labels (optional) - Comma-separated list of label names - + # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` + # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` + # # Example Requests: # GET /issues # GET /issues?state=opened @@ -39,8 +41,7 @@ module API issues = current_user.issues issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues = issues.order('issues.id DESC') - + issues.reorder(issuable_order_by => issuable_sort) present paginate(issues), with: Entities::Issue end end @@ -53,6 +54,8 @@ module API # state (optional) - Return "opened" or "closed" issues # labels (optional) - Comma-separated list of label names # milestone (optional) - Milestone title + # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` + # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` # # Example Requests: # GET /projects/:id/issues @@ -67,11 +70,12 @@ module API issues = user_project.issues issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? + unless params[:milestone].nil? issues = filter_issues_milestone(issues, params[:milestone]) end - issues = issues.order('issues.id DESC') + issues.reorder(issuable_order_by => issuable_sort) present paginate(issues), with: Entities::Issue end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index a365f1db00f..25b7857f4b1 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -25,6 +25,8 @@ module API # Parameters: # id (required) - The ID of a project # state (optional) - Return requests "merged", "opened" or "closed" + # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` + # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` # # Example: # GET /projects/:id/merge_requests @@ -37,25 +39,18 @@ module API # get ":id/merge_requests" do authorize! :read_merge_request, user_project + merge_requests = user_project.merge_requests + + merge_requests = + case params["state"] + when "opened" then merge_requests.opened + when "closed" then merge_requests.closed + when "merged" then merge_requests.merged + else merge_requests + end - mrs = case params["state"] - when "opened" then user_project.merge_requests.opened - when "closed" then user_project.merge_requests.closed - when "merged" then user_project.merge_requests.merged - else user_project.merge_requests - end - - sort = case params["sort"] - when 'desc' then 'DESC' - else 'ASC' - end - - mrs = case params["order_by"] - when 'updated_at' then mrs.order("updated_at #{sort}") - else mrs.order("created_at #{sort}") - end - - present paginate(mrs), with: Entities::MergeRequest + merge_requests.reorder(issuable_order_by => issuable_sort) + present paginate(merge_requests), with: Entities::MergeRequest end # Show MR @@ -75,6 +70,22 @@ module API present merge_request, with: Entities::MergeRequest end + # Show MR changes + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_request/:merge_request_id/changes + # + get ':id/merge_request/:merge_request_id/changes' do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + authorize! :read_merge_request, merge_request + present merge_request, with: Entities::MergeRequestChanges + end + # Create MR # # Parameters: @@ -167,13 +178,9 @@ module API put ":id/merge_request/:merge_request_id/merge" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) - action = if user_project.protected_branch?(merge_request.target_branch) - :push_code_to_protected_branches - else - :push_code - end + allowed = ::Gitlab::GitAccess.can_push_to_branch?(current_user, user_project, merge_request.target_branch) - if can?(current_user, action, user_project) + if allowed if merge_request.unchecked? merge_request.check_if_can_be_merged end @@ -233,7 +240,7 @@ module API if note.save present note, with: Entities::MRNote else - render_validation_error!(note) + render_api_error!("Failed to save note #{note.errors.messages}", 400) end end end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index a4fdb752d69..c5cd73943fb 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -48,7 +48,7 @@ module API if milestone.valid? present milestone, with: Entities::Milestone else - not_found! + render_api_error!("Failed to create milestone #{milestone.errors.messages}", 400) end end @@ -72,9 +72,24 @@ module API if milestone.valid? present milestone, with: Entities::Milestone else - not_found! + render_api_error!("Failed to update milestone #{milestone.errors.messages}", 400) end end + + # Get all issues for a single project milestone + # + # Parameters: + # id (required) - The ID of a project + # milestone_id (required) - The ID of a project milestone + # Example Request: + # GET /projects/:id/milestones/:milestone_id/issues + get ":id/milestones/:milestone_id/issues" do + authorize! :read_milestone, user_project + + @milestone = user_project.milestones.find(params[:milestone_id]) + present paginate(@milestone.issues), with: Entities::Issue + end + end end end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index f9f2ed90ccc..b90ed6af5fb 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -1,10 +1,10 @@ module API # namespaces API class Namespaces < Grape::API - before { + before do authenticate! authenticated_as_admin! - } + end resource :namespaces do # Get a namespaces list diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 0ef9a3c4beb..3726be7c537 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -61,9 +61,42 @@ module API if @note.valid? present @note, with: Entities::Note else - not_found! + not_found!("Note #{@note.errors.messages}") end end + + # Modify existing +noteable+ note + # + # Parameters: + # id (required) - The ID of a project + # noteable_id (required) - The ID of an issue or snippet + # node_id (required) - The ID of a note + # body (required) - New content of a note + # Example Request: + # PUT /projects/:id/issues/:noteable_id/notes/:note_id + # PUT /projects/:id/snippets/:noteable_id/notes/:node_id + put ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do + required_attributes! [:body] + + authorize! :admin_note, user_project.notes.find(params[:note_id]) + + opts = { + note: params[:body], + note_id: params[:note_id], + noteable_type: noteables_str.classify, + noteable_id: params[noteable_id_str] + } + + @note = ::Notes::UpdateService.new(user_project, current_user, + opts).execute + + if @note.valid? + present @note, with: Entities::Note + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end + end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 7d056b9bf58..be9850367b9 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -53,7 +53,7 @@ module API if @hook.errors[:url].present? error!("Invalid url given", 422) end - not_found! + not_found!("Project hook #{@hook.errors.messages}") end end @@ -82,7 +82,7 @@ module API if @hook.errors[:url].present? error!("Invalid url given", 422) end - not_found! + not_found!("Project hook #{@hook.errors.messages}") end end diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index 1595ed0bc36..73cf062155b 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -4,14 +4,6 @@ module API before { authenticate! } resource :projects do - helpers do - def handle_project_member_errors(errors) - if errors[:access_level].any? - error!(errors[:access_level], 422) - end - not_found! - end - end # Get a project team members # @@ -66,7 +58,7 @@ module API @member = team_member.user present @member, with: Entities::ProjectMember, project: user_project else - handle_project_member_errors team_member.errors + handle_member_errors team_member.errors end end @@ -89,7 +81,7 @@ module API @member = team_member.user present @member, with: Entities::ProjectMember, project: user_project else - handle_project_member_errors team_member.errors + handle_member_errors team_member.errors end end @@ -106,7 +98,7 @@ module API unless team_member.nil? team_member.destroy else - {message: "Access revoked", id: params[:user_id].to_i} + { message: "Access revoked", id: params[:user_id].to_i } end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7fcf97d1ad6..0677e85beab 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,23 +11,46 @@ module API attrs[:visibility_level] = Gitlab::VisibilityLevel::PUBLIC if !attrs[:visibility_level].present? && publik == true attrs end + + def filter_projects(projects) + # If the archived parameter is passed, limit results accordingly + if params[:archived].present? + projects = projects.where(archived: parse_boolean(params[:archived])) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + projects.reorder(project_order_by => project_sort) + end + + def project_order_by + order_fields = %w(id name path created_at updated_at last_activity_at) + + if order_fields.include?(params['order_by']) + params['order_by'] + else + 'created_at' + end + end + + def project_sort + if params["sort"] == 'asc' + :asc + else + :desc + end + end end # Get a projects list for authenticated user # - # Parameters: - # archived (optional) - if passed, limit by archived status - # # Example Request: # GET /projects get do @projects = current_user.authorized_projects - - # If the archived parameter is passed, limit results accordingly - if params[:archived].present? - @projects = @projects.where(archived: parse_boolean(params[:archived])) - end - + @projects = filter_projects(@projects) @projects = paginate @projects present @projects, with: Entities::Project end @@ -37,7 +60,9 @@ module API # Example Request: # GET /projects/owned get '/owned' do - @projects = paginate current_user.owned_projects + @projects = current_user.owned_projects + @projects = filter_projects(@projects) + @projects = paginate @projects present @projects, with: Entities::Project end @@ -47,7 +72,9 @@ module API # GET /projects/all get '/all' do authenticated_as_admin! - @projects = paginate Project + @projects = Project.all + @projects = filter_projects(@projects) + @projects = paginate @projects present @projects, with: Entities::Project end @@ -66,7 +93,7 @@ module API # Parameters: # id (required) - The ID of a project # Example Request: - # GET /projects/:id + # GET /projects/:id/events get ":id/events" do limit = (params[:per_page] || 20).to_i offset = (params[:page] || 0).to_i * limit @@ -170,6 +197,49 @@ module API end end + # Update an existing project + # + # Parameters: + # id (required) - the id of a project + # name (optional) - name of a project + # path (optional) - path of a project + # description (optional) - short project description + # issues_enabled (optional) + # merge_requests_enabled (optional) + # wiki_enabled (optional) + # snippets_enabled (optional) + # public (optional) - if true same as setting visibility_level = 20 + # visibility_level (optional) - visibility level of a project + # Example Request + # PUT /projects/:id + put ':id' do + attrs = attributes_for_keys [:name, + :path, + :description, + :default_branch, + :issues_enabled, + :merge_requests_enabled, + :wiki_enabled, + :snippets_enabled, + :public, + :visibility_level] + attrs = map_public_to_visibility_level(attrs) + authorize_admin_project + authorize! :rename_project, user_project if attrs[:name].present? + if attrs[:visibility_level].present? + authorize! :change_visibility_level, user_project + end + + ::Projects::UpdateService.new(user_project, + current_user, attrs).execute + + if user_project.valid? + present user_project, with: Entities::Project + else + render_validation_error!(user_project) + end + end + # Remove project # # Parameters: @@ -198,7 +268,7 @@ module API render_api_error!("Project already forked", 409) end else - not_found! + not_found!("Source Project") end end @@ -227,6 +297,16 @@ module API ids = current_user.authorized_projects.map(&:id) visibility_levels = [ Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PUBLIC ] projects = Project.where("(id in (?) OR visibility_level in (?)) AND (name LIKE (?))", ids, visibility_levels, "%#{params[:query]}%") + sort = params[:sort] == 'desc' ? 'desc' : 'asc' + + projects = case params["order_by"] + when 'id' then projects.order("id #{sort}") + when 'name' then projects.order("name #{sort}") + when 'created_at' then projects.order("created_at #{sort}") + when 'last_activity_at' then projects.order("last_activity_at #{sort}") + else projects + end + present paginate(projects), with: Entities::Project end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index a1a7721b288..b259914a01c 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -58,11 +58,13 @@ module API # ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used # Example Request: # GET /projects/:id/repository/tree - get ":id/repository/tree" do + get ':id/repository/tree' do ref = params[:ref_name] || user_project.try(:default_branch) || 'master' path = params[:path] || nil commit = user_project.repository.commit(ref) + not_found!('Tree') unless commit + tree = user_project.repository.tree(commit.id, path) present tree.sorted_entries, with: Entities::RepoTreeObject @@ -100,14 +102,18 @@ module API # sha (required) - The blob's sha # Example Request: # GET /projects/:id/repository/raw_blobs/:sha - get ":id/repository/raw_blobs/:sha" do + get ':id/repository/raw_blobs/:sha' do ref = params[:sha] repo = user_project.repository - blob = Gitlab::Git::Blob.raw(repo, ref) + begin + blob = Gitlab::Git::Blob.raw(repo, ref) + rescue + not_found! 'Blob' + end - not_found! "Blob" unless blob + not_found! 'Blob' unless blob env['api.format'] = :txt @@ -122,18 +128,28 @@ module API # sha (optional) - the commit sha to download defaults to the tip of the default branch # Example Request: # GET /projects/:id/repository/archive - get ":id/repository/archive", requirements: { format: Gitlab::Regex.archive_formats_regex } do + get ':id/repository/archive', + requirements: { format: Gitlab::Regex.archive_formats_regex } do authorize! :download_code, user_project - file_path = ArchiveRepositoryService.new.execute(user_project, params[:sha], params[:format]) + + begin + file_path = ArchiveRepositoryService.new.execute( + user_project, + params[:sha], + params[:format]) + rescue + not_found!('File') + end if file_path && File.exists?(file_path) data = File.open(file_path, 'rb').read - header["Content-Disposition"] = "attachment; filename=\"#{File.basename(file_path)}\"" + basename = File.basename(file_path) + header['Content-Disposition'] = "attachment; filename=\"#{basename}\"" content_type MIME::Types.type_for(file_path).first.content_type env['api.format'] = :binary present data else - not_found! + not_found!('File') end end @@ -161,7 +177,12 @@ module API get ':id/repository/contributors' do authorize! :download_code, user_project - present user_project.repository.contributors, with: Entities::Contributor + begin + present user_project.repository.contributors, + with: Entities::Contributor + rescue + not_found! + end end end end diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 3e239c5afe7..518964db50d 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -1,10 +1,10 @@ module API # Hooks API class SystemHooks < Grape::API - before { + before do authenticate! authenticated_as_admin! - } + end resource :hooks do # Get the list of system hooks diff --git a/lib/api/users.rb b/lib/api/users.rb index 1a4a8535d48..7c8b3250cd0 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -60,12 +60,18 @@ module API post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :extern_uid, :provider, :bio, :can_create_group, :confirm, :admin] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :can_create_group, :admin, :confirm] user = User.build_user(attrs) admin = attrs.delete(:admin) user.admin = admin unless admin.nil? - confirm = ! (attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) + confirm = !(attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) user.skip_confirmation! unless confirm + + identity_attrs = attributes_for_keys [:provider, :extern_uid] + if identity_attrs.any? + user.identities.build(identity_attrs) + end + if user.save present user, with: Entities::UserFull else @@ -92,8 +98,6 @@ module API # twitter - Twitter account # website_url - Website url # projects_limit - Limit projects each user can create - # extern_uid - External authentication provider UID - # provider - External provider # bio - Bio # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false @@ -102,7 +106,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :extern_uid, :provider, :bio, :can_create_group, :admin] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :can_create_group, :admin] user = User.find(params[:id]) not_found!('User') unless user diff --git a/lib/backup/database.rb b/lib/backup/database.rb index d12d30a9110..9ab6aca276d 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -13,10 +13,10 @@ module Backup def dump success = case config["adapter"] when /^mysql/ then - print "Dumping MySQL database #{config['database']} ... " + $progress.print "Dumping MySQL database #{config['database']} ... " system('mysqldump', *mysql_args, config['database'], out: db_file_name) when "postgresql" then - print "Dumping PostgreSQL database #{config['database']} ... " + $progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env system('pg_dump', config['database'], out: db_file_name) end @@ -27,13 +27,14 @@ module Backup def restore success = case config["adapter"] when /^mysql/ then - print "Restoring MySQL database #{config['database']} ... " + $progress.print "Restoring MySQL database #{config['database']} ... " system('mysql', *mysql_args, config['database'], in: db_file_name) when "postgresql" then - print "Restoring PostgreSQL database #{config['database']} ... " + $progress.print "Restoring PostgreSQL database #{config['database']} ... " # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE # statements like MySQL. Rake::Task["gitlab:db:drop_all_tables"].invoke + Rake::Task["gitlab:db:drop_all_postgres_sequences"].invoke pg_env system('psql', config['database'], '-f', db_file_name) end @@ -68,9 +69,9 @@ module Backup def report_success(success) if success - puts '[DONE]'.green + $progress.puts '[DONE]'.green else - puts '[FAILED]'.red + $progress.puts '[FAILED]'.red end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 03fe0f0b02f..ab8db4e9837 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -18,11 +18,11 @@ module Backup end # create archive - print "Creating backup archive: #{tar_file} ... " + $progress.print "Creating backup archive: #{tar_file} ... " if Kernel.system('tar', '-cf', tar_file, *BACKUP_CONTENTS) - puts "done".green + $progress.puts "done".green else - puts "failed".red + puts "creating archive #{tar_file} failed".red abort 'Backup failed' end @@ -31,37 +31,37 @@ module Backup def upload(tar_file) remote_directory = Gitlab.config.backup.upload.remote_directory - print "Uploading backup archive to remote storage #{remote_directory} ... " + $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection if connection_settings.blank? - puts "skipped".yellow + $progress.puts "skipped".yellow return end connection = ::Fog::Storage.new(connection_settings) directory = connection.directories.get(remote_directory) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false) - puts "done".green + $progress.puts "done".green else - puts "failed".red + puts "uploading backup to #{remote_directory} failed".red abort 'Backup failed' end end def cleanup - print "Deleting tmp directories ... " + $progress.print "Deleting tmp directories ... " if Kernel.system('rm', '-rf', *BACKUP_CONTENTS) - puts "done".green + $progress.puts "done".green else - puts "failed".red + puts "deleting tmp directory failed".red abort 'Backup failed' end end def remove_old # delete backups - print "Deleting old backups ... " + $progress.print "Deleting old backups ... " keep_time = Gitlab.config.backup.keep_time.to_i path = Gitlab.config.backup.path @@ -76,9 +76,9 @@ module Backup end end end - puts "done. (#{removed} removed)".green + $progress.puts "done. (#{removed} removed)".green else - puts "skipping".yellow + $progress.puts "skipping".yellow end end @@ -101,12 +101,12 @@ module Backup exit 1 end - print "Unpacking backup ... " + $progress.print "Unpacking backup ... " unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "failed".red + puts "unpacking backup failed".red exit 1 else - puts "done".green + $progress.puts "done".green end settings = YAML.load_file("backup_information.yml") diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 380beac708d..e18bc804437 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -8,19 +8,21 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - print " * #{project.path_with_namespace} ... " + $progress.print " * #{project.path_with_namespace} ... " # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace if project.empty_repo? - puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".cyan else - output, status = Gitlab::Popen.popen(%W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all)) + cmd = %W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all) + output, status = Gitlab::Popen.popen(cmd) if status.zero? - puts "[DONE]".green + $progress.puts "[DONE]".green else puts "[FAILED]".red + puts "failed: #{cmd.join(' ')}" puts output abort 'Backup failed' end @@ -29,15 +31,17 @@ module Backup wiki = ProjectWiki.new(project) if File.exists?(path_to_repo(wiki)) - print " * #{wiki.path_with_namespace} ... " + $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? - puts " [SKIPPED]".cyan + $progress.puts " [SKIPPED]".cyan else - output, status = Gitlab::Popen.popen(%W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)) + cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + output, status = Gitlab::Popen.popen(cmd) if status.zero? - puts " [DONE]".green + $progress.puts " [DONE]".green else puts " [FAILED]".red + puts "failed: #{cmd.join(' ')}" abort 'Backup failed' end end @@ -55,35 +59,52 @@ module Backup FileUtils.mkdir_p(repos_path) Project.find_each(batch_size: 1000) do |project| - print "#{project.path_with_namespace} ... " + $progress.print " * #{project.path_with_namespace} ... " project.namespace.ensure_dir_exist if project.namespace - if system(*%W(git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)}), silent) - puts "[DONE]".green + if File.exists?(path_to_bundle(project)) + cmd = %W(git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)}) + else + cmd = %W(git init --bare #{path_to_repo(project)}) + end + + if system(*cmd, silent) + $progress.puts "[DONE]".green else puts "[FAILED]".red + puts "failed: #{cmd.join(' ')}" abort 'Restore failed' end wiki = ProjectWiki.new(project) if File.exists?(path_to_bundle(wiki)) - print " * #{wiki.path_with_namespace} ... " - if system(*%W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}), silent) - puts " [DONE]".green + $progress.print " * #{wiki.path_with_namespace} ... " + + # If a wiki bundle exists, first remove the empty repo + # that was initialized with ProjectWiki.new() and then + # try to restore with 'git clone --bare'. + FileUtils.rm_rf(path_to_repo(wiki)) + cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + + if system(*cmd, silent) + $progress.puts " [DONE]".green else puts " [FAILED]".red + puts "failed: #{cmd.join(' ')}" abort 'Restore failed' end end end - print 'Put GitLab hooks in repositories dirs'.yellow - if system("#{Gitlab.config.gitlab_shell.path}/bin/create-hooks") - puts " [DONE]".green + $progress.print 'Put GitLab hooks in repositories dirs'.yellow + cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" + if system(cmd) + $progress.puts " [DONE]".green else puts " [FAILED]".red + puts "failed: #{cmd}" end end @@ -91,7 +112,7 @@ module Backup protected def path_to_repo(project) - File.join(repos_path, project.path_with_namespace + '.git') + project.repository.path_to_repo end def path_to_bundle(project) diff --git a/lib/email_validator.rb b/lib/email_validator.rb index 0a67ebcd795..f509f0a5843 100644 --- a/lib/email_validator.rb +++ b/lib/email_validator.rb @@ -1,5 +1,5 @@ # Based on https://github.com/balexand/email_validator -# +# # Extended to use only strict mode with following allowed characters: # ' - apostrophe # diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index e51cb30bdd9..6e4ed01e079 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -1,17 +1,9 @@ # Module providing methods for dealing with separating a tree-ish string and a # file path string when combined in a request parameter module ExtractsPath - extend ActiveSupport::Concern - # Raised when given an invalid file path class InvalidPathError < StandardError; end - included do - if respond_to?(:before_filter) - before_filter :assign_ref_vars - end - end - # Given a string containing both a Git tree-ish, such as a branch or tag, and # a filesystem path joined by forward slashes, attempts to separate the two. # @@ -110,7 +102,8 @@ module ExtractsPath raise InvalidPathError unless @commit @hex_path = Digest::SHA1.hexdigest(@path) - @logs_path = logs_file_project_ref_path(@project, @ref, @path) + @logs_path = logs_file_namespace_project_ref_path(@project.namespace, + @project, @ref, @path) rescue RuntimeError, NoMethodError, InvalidPathError not_found! diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 411b2b9a3cc..424541b4a04 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -11,6 +11,11 @@ module Gitlab MASTER = 40 OWNER = 50 + # Branch protection settings + PROTECTION_NONE = 0 + PROTECTION_DEV_CAN_PUSH = 1 + PROTECTION_FULL = 2 + class << self def values options.values @@ -43,6 +48,18 @@ module Gitlab master: MASTER, } end + + def protection_options + { + "Not protected, developers and masters can (force) push and delete the branch" => PROTECTION_NONE, + "Partially protected, developers can also push but prevent all force pushes and deletion" => PROTECTION_DEV_CAN_PUSH, + "Fully protected, only masters can push and prevent all force pushes and deletion" => PROTECTION_FULL, + } + end + + def protection_values + protection_options.values + end end def human_access diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index ae33c529b93..30509528b8b 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,7 +1,7 @@ module Gitlab class Auth def find(login, password) - user = User.find_by(email: login) || User.find_by(username: login) + user = User.by_login(login) # If no user is found, or it's an LDAP server, try LDAP. # LDAP users are only authenticated via LDAP diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index df1461a45c9..dc4b945f9d4 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -71,16 +71,46 @@ module Grack false end + def oauth_access_token_check(login, password) + if login == "oauth2" && git_cmd == 'git-upload-pack' && password.present? + token = Doorkeeper::AccessToken.by_token(password) + token && token.accessible? && User.find_by(id: token.resource_owner_id) + end + end + def authenticate_user(login, password) - auth = Gitlab::Auth.new - auth.find(login, password) + user = Gitlab::Auth.new.find(login, password) + + unless user + user = oauth_access_token_check(login, password) + end + + return user if user.present? + + # At this point, we know the credentials were wrong. We let Rack::Attack + # know there was a failed authentication attempt from this IP. This + # information is stored in the Rails cache (Redis) and will be used by + # the Rack::Attack middleware to decide whether to block requests from + # this IP. + config = Gitlab.config.rack_attack.git_basic_auth + Rack::Attack::Allow2Ban.filter(@request.ip, config) do + # Unless the IP is whitelisted, return true so that Allow2Ban + # increments the counter (stored in Rails.cache) for the IP + if config.ip_whitelist.include?(@request.ip) + false + else + true + end + end + + nil # No user was found end def authorized_request? case git_cmd when *Gitlab::GitAccess::DOWNLOAD_COMMANDS if user - Gitlab::GitAccess.new.download_allowed?(user, project) + Gitlab::GitAccess.new.download_access_check(user, project).allowed? elsif project.public? # Allow clone/fetch for public projects true @@ -119,12 +149,13 @@ module Grack path_with_namespace = m.last path_with_namespace.gsub!(/\.wiki$/, '') + path_with_namespace[0] = '' if path_with_namespace.start_with?('/') Project.find_with_namespace(path_with_namespace) end end def render_not_found - [404, {"Content-Type" => "text/plain"}, ["Not Found"]] + [404, { "Content-Type" => "text/plain" }, ["Not Found"]] end end end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index ddb1ac61bf5..aabc7f1e69a 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -8,6 +8,13 @@ module Gitlab end end + class << self + def version_required + @version_required ||= File.read(Rails.root. + join('GITLAB_SHELL_VERSION')).strip + end + end + # Init new repository # # name - project path with namespace @@ -16,7 +23,8 @@ module Gitlab # add_repository("gitlab/gitlab-ci") # def add_repository(name) - system gitlab_shell_projects_path, 'add-project', "#{name}.git" + Gitlab::Utils.system_silent([gitlab_shell_projects_path, + 'add-project', "#{name}.git"]) end # Import repository @@ -27,7 +35,8 @@ module Gitlab # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(name, url) - system gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240' + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'import-project', + "#{name}.git", url, '240']) end # Move repository @@ -39,7 +48,8 @@ module Gitlab # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new.git") # def mv_repository(path, new_path) - system gitlab_shell_projects_path, 'mv-project', "#{path}.git", "#{new_path}.git" + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', + "#{path}.git", "#{new_path}.git"]) end # Update HEAD for repository @@ -51,7 +61,8 @@ module Gitlab # update_repository_head("gitlab/gitlab-ci", "3-1-stable") # def update_repository_head(path, branch) - system gitlab_shell_projects_path, 'update-head', "#{path}.git", branch + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'update-head', + "#{path}.git", branch]) end # Fork repository to new namespace @@ -63,7 +74,8 @@ module Gitlab # fork_repository("gitlab/gitlab-ci", "randx") # def fork_repository(path, fork_namespace) - system gitlab_shell_projects_path, 'fork-project', "#{path}.git", fork_namespace + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', + "#{path}.git", fork_namespace]) end # Remove repository from file system @@ -74,7 +86,8 @@ module Gitlab # remove_repository("gitlab/gitlab-ci") # def remove_repository(name) - system gitlab_shell_projects_path, 'rm-project', "#{name}.git" + Gitlab::Utils.system_silent([gitlab_shell_projects_path, + 'rm-project', "#{name}.git"]) end # Add repository branch from passed ref @@ -87,7 +100,8 @@ module Gitlab # add_branch("gitlab/gitlab-ci", "4-0-stable", "master") # def add_branch(path, branch_name, ref) - system gitlab_shell_projects_path, 'create-branch', "#{path}.git", branch_name, ref + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'create-branch', + "#{path}.git", branch_name, ref]) end # Remove repository branch @@ -99,7 +113,8 @@ module Gitlab # rm_branch("gitlab/gitlab-ci", "4-0-stable") # def rm_branch(path, branch_name) - system gitlab_shell_projects_path, 'rm-branch', "#{path}.git", branch_name + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'rm-branch', + "#{path}.git", branch_name]) end # Add repository tag from passed ref @@ -117,7 +132,7 @@ module Gitlab cmd = %W(#{gitlab_shell_path}/bin/gitlab-projects create-tag #{path}.git #{tag_name} #{ref}) cmd << message unless message.nil? || message.empty? - system *cmd + Gitlab::Utils.system_silent(cmd) end # Remove repository tag @@ -129,7 +144,8 @@ module Gitlab # rm_tag("gitlab/gitlab-ci", "v4.0") # def rm_tag(path, tag_name) - system gitlab_shell_projects_path, 'rm-tag', "#{path}.git", tag_name + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'rm-tag', + "#{path}.git", tag_name]) end # Add new key to gitlab-shell @@ -138,7 +154,8 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) - system gitlab_shell_keys_path, 'add-key', key_id, key_content + Gitlab::Utils.system_silent([gitlab_shell_keys_path, + 'add-key', key_id, key_content]) end # Batch-add keys to authorized_keys @@ -157,7 +174,8 @@ module Gitlab # remove_key("key-342", "sha-rsa ...") # def remove_key(key_id, key_content) - system gitlab_shell_keys_path, 'rm-key', key_id, key_content + Gitlab::Utils.system_silent([gitlab_shell_keys_path, + 'rm-key', key_id, key_content]) end # Remove all ssh keys from gitlab shell @@ -166,7 +184,7 @@ module Gitlab # remove_all_keys # def remove_all_keys - system gitlab_shell_keys_path, 'clear' + Gitlab::Utils.system_silent([gitlab_shell_keys_path, 'clear']) end # Add empty directory for storing repositories diff --git a/lib/gitlab/backend/shell_adapter.rb b/lib/gitlab/backend/shell_adapter.rb index f247f4593d7..fbe2a7a0d72 100644 --- a/lib/gitlab/backend/shell_adapter.rb +++ b/lib/gitlab/backend/shell_adapter.rb @@ -9,4 +9,3 @@ module Gitlab end end end - diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb new file mode 100644 index 00000000000..7298152e7e9 --- /dev/null +++ b/lib/gitlab/bitbucket_import.rb @@ -0,0 +1,6 @@ +module Gitlab + module BitbucketImport + mattr_accessor :public_key + @public_key = nil + end +end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb new file mode 100644 index 00000000000..c907bebaef6 --- /dev/null +++ b/lib/gitlab/bitbucket_import/client.rb @@ -0,0 +1,99 @@ +module Gitlab + module BitbucketImport + class Client + attr_reader :consumer, :api + + def initialize(access_token = nil, access_token_secret = nil) + @consumer = ::OAuth::Consumer.new( + config.app_id, + config.app_secret, + bitbucket_options + ) + + if access_token && access_token_secret + @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) + end + end + + def request_token(redirect_uri) + request_token = consumer.get_request_token(oauth_callback: redirect_uri) + + { + oauth_token: request_token.token, + oauth_token_secret: request_token.secret, + oauth_callback_confirmed: request_token.callback_confirmed?.to_s + } + end + + def authorize_url(request_token, redirect_uri) + request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) + + if request_token.callback_confirmed? + request_token.authorize_url + else + request_token.authorize_url(oauth_callback: redirect_uri) + end + end + + def get_token(request_token, oauth_verifier, redirect_uri) + request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) + + if request_token.callback_confirmed? + request_token.get_access_token(oauth_verifier: oauth_verifier) + else + request_token.get_access_token(oauth_callback: redirect_uri) + end + end + + def user + JSON.parse(api.get("/api/1.0/user").body) + end + + def issues(project_identifier) + JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues").body) + end + + def issue_comments(project_identifier, issue_id) + JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) + end + + def project(project_identifier) + JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}").body) + end + + def find_deploy_key(project_identifier, key) + JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| + deploy_key["key"].chomp == key.chomp + end + end + + def add_deploy_key(project_identifier, key) + deploy_key = find_deploy_key(project_identifier, key) + return if deploy_key + + JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) + end + + def delete_deploy_key(project_identifier, key) + deploy_key = find_deploy_key(project_identifier, key) + return unless deploy_key + + api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" + end + + def projects + JSON.parse(api.get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } + end + + private + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"} + end + + def bitbucket_options + OmniAuth::Strategies::Bitbucket.default_options[:client_options] + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb new file mode 100644 index 00000000000..42c93707caa --- /dev/null +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -0,0 +1,52 @@ +module Gitlab + module BitbucketImport + class Importer + attr_reader :project, :client + + def initialize(project) + @project = project + @client = Client.new(project.creator.bitbucket_access_token, project.creator.bitbucket_access_token_secret) + @formatter = Gitlab::ImportFormatter.new + end + + def execute + project_identifier = project.import_source + + return true unless client.project(project_identifier)["has_issues"] + + #Issues && Comments + issues = client.issues(project_identifier) + + issues["issues"].each do |issue| + body = @formatter.author_line(issue["reported_by"]["username"], issue["content"]) + + comments = client.issue_comments(project_identifier, issue["local_id"]) + + if comments.any? + body += @formatter.comments_header + end + + comments.each do |comment| + body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"]) + end + + project.issues.create!( + description: body, + title: issue["title"], + state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened', + author_id: gl_user_id(project, issue["reported_by"]["username"]) + ) + end + + true + end + + private + + def gl_user_id(project, bitbucket_id) + user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + (user && user.id) || project.creator_id + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb new file mode 100644 index 00000000000..9931aa7e029 --- /dev/null +++ b/lib/gitlab/bitbucket_import/key_adder.rb @@ -0,0 +1,23 @@ +module Gitlab + module BitbucketImport + class KeyAdder + attr_reader :repo, :current_user, :client + + def initialize(repo, current_user) + @repo, @current_user = repo, current_user + @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret) + end + + def execute + return false unless BitbucketImport.public_key.present? + + project_identifier = "#{repo["owner"]}/#{repo["slug"]}" + client.add_deploy_key(project_identifier, BitbucketImport.public_key) + + true + rescue + false + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb new file mode 100644 index 00000000000..1a24a86fc37 --- /dev/null +++ b/lib/gitlab/bitbucket_import/key_deleter.rb @@ -0,0 +1,23 @@ +module Gitlab + module BitbucketImport + class KeyDeleter + attr_reader :project, :current_user, :client + + def initialize(project) + @project = project + @current_user = project.creator + @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret) + end + + def execute + return false unless BitbucketImport.public_key.present? + + client.delete_deploy_key(project.import_source, BitbucketImport.public_key) + + true + rescue + false + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb new file mode 100644 index 00000000000..db33af2c2da --- /dev/null +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -0,0 +1,39 @@ +module Gitlab + module BitbucketImport + class ProjectCreator + attr_reader :repo, :namespace, :current_user + + def initialize(repo, namespace, current_user) + @repo = repo + @namespace = namespace + @current_user = current_user + end + + def execute + @project = Project.new( + name: repo["name"], + path: repo["slug"], + description: repo["description"], + namespace: namespace, + creator: current_user, + visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, + import_type: "bitbucket", + import_source: "#{repo["owner"]}/#{repo["slug"]}", + import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git" + ) + + if @project.save! + @project.reload + + if @project.import_failed? + @project.import_retry + else + @project.import_start + end + end + + @project + end + end + end +end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 401e6e047b1..a9fd59f03d9 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -3,14 +3,19 @@ module Gitlab ISSUE_CLOSING_REGEX = Regexp.new(Gitlab.config.gitlab.issue_closing_pattern) def self.closed_by_message_in_project(message, project) - md = ISSUE_CLOSING_REGEX.match(message) - if md - extractor = Gitlab::ReferenceExtractor.new - extractor.analyze(md[0], project) - extractor.issues_for(project) - else - [] + issues = [] + + unless message.nil? + md = message.scan(ISSUE_CLOSING_REGEX) + + md.each do |ref| + extractor = Gitlab::ReferenceExtractor.new + extractor.analyze(ref[0], project) + issues += extractor.issues_for(project) + end end + + issues.uniq end end end diff --git a/lib/gitlab/commits_calendar.rb b/lib/gitlab/commits_calendar.rb new file mode 100644 index 00000000000..2f30d238e6b --- /dev/null +++ b/lib/gitlab/commits_calendar.rb @@ -0,0 +1,33 @@ +module Gitlab + class CommitsCalendar + attr_reader :timestamps + + def initialize(projects, user) + @timestamps = {} + date_timestamps = [] + + projects.reject(&:forked?).each do |project| + date_timestamps << ProjectContributions.new(project, user).commits_log + end + + # Sumarrize commits from all projects per days + date_timestamps = date_timestamps.inject do |collection, date| + collection.merge(date) { |k, old_v, new_v| old_v + new_v } + end + + date_timestamps ||= [] + date_timestamps.each do |date, commits| + timestamp = Date.parse(date).to_time.to_i.to_s rescue nil + @timestamps[timestamp] = commits if timestamp + end + end + + def starting_year + (Time.now - 1.year).strftime("%Y") + end + + def starting_month + Date.today.strftime("%m").to_i + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb new file mode 100644 index 00000000000..1a25eebe7d1 --- /dev/null +++ b/lib/gitlab/current_settings.rb @@ -0,0 +1,27 @@ +module Gitlab + module CurrentSettings + def current_application_settings + key = :current_application_settings + + RequestStore.store[key] ||= begin + if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('application_settings') + RequestStore.store[:current_application_settings] = + (ApplicationSetting.current || ApplicationSetting.create_from_defaults) + else + fake_application_settings + end + end + end + + def fake_application_settings + OpenStruct.new( + default_projects_limit: Settings.gitlab['default_projects_limit'], + default_branch_protection: Settings.gitlab['default_branch_protection'], + signup_enabled: Settings.gitlab['signup_enabled'], + signin_enabled: Settings.gitlab['signin_enabled'], + gravatar_enabled: Settings.gravatar['enabled'], + sign_in_text: Settings.extra['sign_in_text'], + ) + end + end +end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index f7c1f20d762..c1d9520ddf1 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -4,7 +4,7 @@ module Gitlab include Enumerable def parse(lines) - @lines = lines, + @lines = lines lines_obj = [] line_obj_index = 0 line_old = 1 @@ -27,7 +27,7 @@ module Gitlab line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 - next if line_old == 1 && line_new == 1 #top of file + next if line_old <= 1 && line_new <= 1 #top of file lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next @@ -74,7 +74,7 @@ module Gitlab def html_escape(str) replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } - str.gsub(/[&"'><]/, replacements) + str.gsub(/[&"'><]/, replacements) end end end diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb new file mode 100644 index 00000000000..eae9773a067 --- /dev/null +++ b/lib/gitlab/force_push_check.rb @@ -0,0 +1,14 @@ +module Gitlab + class ForcePushCheck + def self.force_push?(project, oldrev, newrev) + return false if project.empty_repo? + + if oldrev != Gitlab::Git::BLANK_SHA && newrev != Gitlab::Git::BLANK_SHA + missed_refs, _ = Gitlab::Popen.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})) + missed_refs.split("\n").size > 0 + else + false + end + end + end +end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb new file mode 100644 index 00000000000..4a712c6345f --- /dev/null +++ b/lib/gitlab/git.rb @@ -0,0 +1,9 @@ +module Gitlab + module Git + BLANK_SHA = '0' * 40 + + def self.extract_ref_name(ref) + ref.gsub(/\Arefs\/(tags|heads)\//, '') + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index b768a99a0e8..9b31190a882 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -5,95 +5,129 @@ module Gitlab attr_reader :params, :project, :git_cmd, :user - def allowed?(actor, cmd, project, changes = nil) + def self.can_push_to_branch?(user, project, ref) + return false unless user + + if project.protected_branch?(ref) && + !(project.developers_can_push_to_protected_branch?(ref) && project.team.developer?(user)) + user.can?(:push_code_to_protected_branches, project) + else + user.can?(:push_code, project) + end + end + + def check(actor, cmd, project, changes = nil) case cmd when *DOWNLOAD_COMMANDS + download_access_check(actor, project) + when *PUSH_COMMANDS if actor.is_a? User - download_allowed?(actor, project) + push_access_check(actor, project, changes) elsif actor.is_a? DeployKey - actor.projects.include?(project) + return build_status_object(false, "Deploy key not allowed to push") elsif actor.is_a? Key - download_allowed?(actor.user, project) + push_access_check(actor.user, project, changes) else raise 'Wrong actor' end - when *PUSH_COMMANDS - if actor.is_a? User - push_allowed?(actor, project, changes) - elsif actor.is_a? DeployKey - # Deploy key not allowed to push - return false - elsif actor.is_a? Key - push_allowed?(actor.user, project, changes) + else + return build_status_object(false, "Wrong command") + end + end + + def download_access_check(actor, project) + if actor.is_a?(User) + user_download_access_check(actor, project) + elsif actor.is_a?(DeployKey) + if actor.projects.include?(project) + build_status_object(true) else - raise 'Wrong actor' + build_status_object(false, "Deploy key not allowed to access this project") end + elsif actor.is_a? Key + user_download_access_check(actor.user, project) else - false + raise 'Wrong actor' end end - def download_allowed?(user, project) - if user && user_allowed?(user) - user.can?(:download_code, project) + def user_download_access_check(user, project) + if user && user_allowed?(user) && user.can?(:download_code, project) + build_status_object(true) else - false + build_status_object(false, "You don't have access") end end - def push_allowed?(user, project, changes) - return false unless user && user_allowed?(user) - return true if changes.blank? + def push_access_check(user, project, changes) + unless user && user_allowed?(user) + return build_status_object(false, "You don't have access") + end + + if changes.blank? + return build_status_object(true) + end + + unless project.repository.exists? + return build_status_object(false, "Repository does not exist") + end changes = changes.lines if changes.kind_of?(String) # Iterate over all changes to find if user allowed all of them to be applied - changes.each do |change| - unless change_allowed?(user, project, change) + changes.map(&:strip).reject(&:blank?).each do |change| + status = change_access_check(user, project, change) + unless status.allowed? # If user does not have access to make at least one change - cancel all push - return false + return status end end - # If user has access to make all changes - true + return build_status_object(true) end - def change_allowed?(user, project, change) + def change_access_check(user, project, change) oldrev, newrev, ref = change.split(' ') action = if project.protected_branch?(branch_name(ref)) - # we dont allow force push to protected branch - if forced_push?(project, oldrev, newrev) - :force_push_code_to_protected_branches - # and we dont allow remove of protected branch - elsif newrev =~ /0000000/ - :remove_protected_branches - else - :push_code_to_protected_branches - end - elsif project.repository && project.repository.tag_names.include?(tag_name(ref)) + protected_branch_action(project, oldrev, newrev, branch_name(ref)) + elsif protected_tag?(project, tag_name(ref)) # Prevent any changes to existing git tag unless user has permissions :admin_project else :push_code end - user.can?(action, project) + if user.can?(action, project) + build_status_object(true) + else + build_status_object(false, "You don't have permission") + end end def forced_push?(project, oldrev, newrev) - return false if project.empty_repo? + Gitlab::ForcePushCheck.force_push?(project, oldrev, newrev) + end - if oldrev !~ /00000000/ && newrev !~ /00000000/ - missed_refs = IO.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})).read - missed_refs.split("\n").size > 0 + private + + def protected_branch_action(project, oldrev, newrev, branch_name) + # we dont allow force push to protected branch + if forced_push?(project, oldrev, newrev) + :force_push_code_to_protected_branches + elsif newrev == Gitlab::Git::BLANK_SHA + # and we dont allow remove of protected branch + :remove_protected_branches + elsif project.developers_can_push_to_protected_branch?(branch_name) + :push_code else - false + :push_code_to_protected_branches end end - private + def protected_tag?(project, tag_name) + project.repository.tag_names.include?(tag_name) + end def user_allowed?(user) Gitlab::UserAccess.allowed?(user) @@ -116,5 +150,11 @@ module Gitlab nil end end + + protected + + def build_status_object(status, message = '') + GitAccessStatus.new(status, message) + end end end diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb new file mode 100644 index 00000000000..5a806ff6e0d --- /dev/null +++ b/lib/gitlab/git_access_status.rb @@ -0,0 +1,15 @@ +module Gitlab + class GitAccessStatus + attr_accessor :status, :message + alias_method :allowed?, :status + + def initialize(status, message = '') + @status = status + @message = message + end + + def to_json + { status: @status, message: @message }.to_json + end + end +end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 9f0eb3be20f..a2177c8d548 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,7 +1,11 @@ module Gitlab class GitAccessWiki < GitAccess - def change_allowed?(user, project, change) - user.can?(:write_wiki, project) + def change_access_check(user, project, change) + if user.can?(:write_wiki, project) + build_status_object(true) + else + build_status_object(false, "You don't have access") + end end end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 13cb08948bb..39d17def930 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -5,7 +5,8 @@ module Gitlab # # Returns true for a valid reference name, false otherwise def validate(ref_name) - system *%W(git check-ref-format refs/#{ref_name}) + Gitlab::Utils.system_silent( + %W(git check-ref-format refs/#{ref_name})) end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb new file mode 100644 index 00000000000..676d226bddd --- /dev/null +++ b/lib/gitlab/github_import/client.rb @@ -0,0 +1,53 @@ +module Gitlab + module GithubImport + class Client + attr_reader :client, :api + + def initialize(access_token) + @client = ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options + ) + + if access_token + ::Octokit.auto_paginate = true + @api = ::Octokit::Client.new(access_token: access_token) + end + end + + def authorize_url(redirect_uri) + client.auth_code.authorize_url({ + redirect_uri: redirect_uri, + scope: "repo, user, user:email" + }) + end + + def get_token(code) + client.auth_code.get_token(code).token + end + + def method_missing(method, *args, &block) + if api.respond_to?(method) + api.send(method, *args, &block) + else + super(method, *args, &block) + end + end + + def respond_to?(method) + api.respond_to?(method) || super + end + + private + + def config + Gitlab.config.omniauth.providers.find{|provider| provider.name == "github"} + end + + def github_options + OmniAuth::Strategies::GitHub.default_options[:client_options] + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb new file mode 100644 index 00000000000..23832b3233c --- /dev/null +++ b/lib/gitlab/github_import/importer.rb @@ -0,0 +1,46 @@ +module Gitlab + module GithubImport + class Importer + attr_reader :project, :client + + def initialize(project) + @project = project + @client = Client.new(project.creator.github_access_token) + @formatter = Gitlab::ImportFormatter.new + end + + def execute + #Issues && Comments + client.list_issues(project.import_source, state: :all).each do |issue| + if issue.pull_request.nil? + + body = @formatter.author_line(issue.user.login, issue.body) + + if issue.comments > 0 + body += @formatter.comments_header + + client.issue_comments(project.import_source, issue.number).each do |c| + body += @formatter.comment(c.user.login, c.created_at, c.body) + end + end + + project.issues.create!( + description: body, + title: issue.title, + state: issue.state == 'closed' ? 'closed' : 'opened', + author_id: gl_user_id(project, issue.user.id) + ) + end + end + end + + private + + def gl_user_id(project, github_id) + user = User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s) + (user && user.id) || project.creator_id + end + end + end +end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb new file mode 100644 index 00000000000..9439ca6cbf4 --- /dev/null +++ b/lib/gitlab/github_import/project_creator.rb @@ -0,0 +1,39 @@ +module Gitlab + module GithubImport + class ProjectCreator + attr_reader :repo, :namespace, :current_user + + def initialize(repo, namespace, current_user) + @repo = repo + @namespace = namespace + @current_user = current_user + end + + def execute + @project = Project.new( + name: repo.name, + path: repo.name, + description: repo.description, + namespace: namespace, + creator: current_user, + visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, + import_type: "github", + import_source: repo.full_name, + import_url: repo.clone_url.sub("https://", "https://#{current_user.github_access_token}@") + ) + + if @project.save! + @project.reload + + if @project.import_failed? + @project.import_retry + else + @project.import_start + end + end + + @project + end + end + end +end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb new file mode 100644 index 00000000000..ecf4ff94e39 --- /dev/null +++ b/lib/gitlab/gitlab_import/client.rb @@ -0,0 +1,78 @@ +module Gitlab + module GitlabImport + class Client + attr_reader :client, :api + + PER_PAGE = 100 + + def initialize(access_token) + @client = ::OAuth2::Client.new( + config.app_id, + config.app_secret, + gitlab_options + ) + + if access_token + @api = OAuth2::AccessToken.from_hash(@client, access_token: access_token) + end + end + + def authorize_url(redirect_uri) + client.auth_code.authorize_url({ + redirect_uri: redirect_uri, + scope: "api" + }) + end + + def get_token(code, redirect_uri) + client.auth_code.get_token(code, redirect_uri: redirect_uri).token + end + + def issues(project_identifier) + lazy_page_iterator(PER_PAGE) do |page| + api.get("/api/v3/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed + end + end + + def issue_comments(project_identifier, issue_id) + lazy_page_iterator(PER_PAGE) do |page| + api.get("/api/v3/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed + end + end + + def project(id) + api.get("/api/v3/projects/#{id}").parsed + end + + def projects + lazy_page_iterator(PER_PAGE) do |page| + api.get("/api/v3/projects?per_page=#{PER_PAGE}&page=#{page}").parsed + end + end + + private + + def lazy_page_iterator(per_page) + Enumerator.new do |y| + page = 1 + loop do + items = yield(page) + items.each do |item| + y << item + end + break if items.empty? || items.size < per_page + page += 1 + end + end + end + + def config + Gitlab.config.omniauth.providers.find{|provider| provider.name == "gitlab"} + end + + def gitlab_options + OmniAuth::Strategies::GitLab.default_options[:client_options] + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb new file mode 100644 index 00000000000..c5304a0699b --- /dev/null +++ b/lib/gitlab/gitlab_import/importer.rb @@ -0,0 +1,50 @@ +module Gitlab + module GitlabImport + class Importer + attr_reader :project, :client + + def initialize(project) + @project = project + @client = Client.new(project.creator.gitlab_access_token) + @formatter = Gitlab::ImportFormatter.new + end + + def execute + project_identifier = URI.encode(project.import_source, '/') + + #Issues && Comments + issues = client.issues(project_identifier) + + issues.each do |issue| + body = @formatter.author_line(issue["author"]["name"], issue["description"]) + + comments = client.issue_comments(project_identifier, issue["id"]) + + if comments.any? + body += @formatter.comments_header + end + + comments.each do |comment| + body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) + end + + project.issues.create!( + description: body, + title: issue["title"], + state: issue["state"], + author_id: gl_user_id(project, issue["author"]["id"]) + ) + end + + true + end + + private + + def gl_user_id(project, gitlab_id) + user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) + (user && user.id) || project.creator_id + end + end + end +end diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb new file mode 100644 index 00000000000..6424d56f8f1 --- /dev/null +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -0,0 +1,39 @@ +module Gitlab + module GitlabImport + class ProjectCreator + attr_reader :repo, :namespace, :current_user + + def initialize(repo, namespace, current_user) + @repo = repo + @namespace = namespace + @current_user = current_user + end + + def execute + @project = Project.new( + name: repo["name"], + path: repo["path"], + description: repo["description"], + namespace: namespace, + creator: current_user, + visibility_level: repo["visibility_level"], + import_type: "gitlab", + import_source: repo["path_with_namespace"], + import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{current_user.gitlab_access_token}@") + ) + + if @project.save! + @project.reload + + if @project.import_failed? + @project.import_retry + else + @project.import_start + end + end + + @project + end + end + end +end diff --git a/lib/gitlab/gitorious_import/client.rb b/lib/gitlab/gitorious_import/client.rb new file mode 100644 index 00000000000..5043f6a2ebd --- /dev/null +++ b/lib/gitlab/gitorious_import/client.rb @@ -0,0 +1,63 @@ +module Gitlab + module GitoriousImport + GITORIOUS_HOST = "https://gitorious.org" + + class Client + attr_reader :repo_list + + def initialize(repo_list) + @repo_list = repo_list + end + + def authorize_url(redirect_uri) + "#{GITORIOUS_HOST}/gitlab-import?callback_url=#{redirect_uri}" + end + + def repos + @repos ||= repo_names.map { |full_name| Repository.new(full_name) } + end + + def repo(id) + repos.find { |repo| repo.id == id } + end + + private + + def repo_names + repo_list.to_s.split(',').map(&:strip).reject(&:blank?) + end + end + + Repository = Struct.new(:full_name) do + def id + Digest::SHA1.hexdigest(full_name) + end + + def namespace + segments.first + end + + def path + segments.last + end + + def name + path.titleize + end + + def description + "" + end + + def import_url + "#{GITORIOUS_HOST}/#{full_name}.git" + end + + private + + def segments + full_name.split('/') + end + end + end +end diff --git a/lib/gitlab/gitorious_import/project_creator.rb b/lib/gitlab/gitorious_import/project_creator.rb new file mode 100644 index 00000000000..3cbebe53997 --- /dev/null +++ b/lib/gitlab/gitorious_import/project_creator.rb @@ -0,0 +1,39 @@ +module Gitlab + module GitoriousImport + class ProjectCreator + attr_reader :repo, :namespace, :current_user + + def initialize(repo, namespace, current_user) + @repo = repo + @namespace = namespace + @current_user = current_user + end + + def execute + @project = Project.new( + name: repo.name, + path: repo.path, + description: repo.description, + namespace: namespace, + creator: current_user, + visibility_level: Gitlab::VisibilityLevel::PUBLIC, + import_type: "gitorious", + import_source: repo.full_name, + import_url: repo.import_url + ) + + if @project.save! + @project.reload + + if @project.import_failed? + @project.import_retry + else + @project.import_start + end + end + + @project + end + end + end +end diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb new file mode 100644 index 00000000000..72e041a90b1 --- /dev/null +++ b/lib/gitlab/import_formatter.rb @@ -0,0 +1,15 @@ +module Gitlab + class ImportFormatter + def comment(author, date, body) + "\n\n*By #{author} on #{date}*\n\n#{body}" + end + + def comments_header + "\n\n\n**Imported comments:**\n" + end + + def author_line(author, body) + "*Created by: #{author}*\n\n#{body}" + end + end +end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index eb2c4e48ff2..0c85acf7e69 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :adapter, :provider, :user def self.open(user, &block) - Gitlab::LDAP::Adapter.open(user.provider) do |adapter| + Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| block.call(self.new(user, adapter)) end end @@ -28,13 +28,13 @@ module Gitlab def initialize(user, adapter=nil) @adapter = adapter @user = user - @provider = user.provider + @provider = user.ldap_identity.provider end def allowed? - if Gitlab::LDAP::Person.find_by_dn(user.extern_uid, adapter) + if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) return true unless ldap_config.active_directory - !Gitlab::LDAP::Person.disabled_via_active_directory?(user.extern_uid, adapter) + !Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) else false end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 256cdb4c2f1..577a890a7d9 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -63,8 +63,10 @@ module Gitlab end def dn_matches_filter?(dn, filter) - ldap_search(base: dn, filter: filter, - scope: Net::LDAP::SearchScope_BaseObject, attributes: %w{dn}).any? + ldap_search(base: dn, + filter: filter, + scope: Net::LDAP::SearchScope_BaseObject, + attributes: %w{dn}).any? end def ldap_search(*args) diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 3176e9790a7..cfa8692659d 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -12,9 +12,10 @@ module Gitlab class << self def find_by_uid_and_provider(uid, provider) # LDAP distinguished name is case-insensitive - ::User. + identity = ::Identity. where(provider: [provider, :ldap]). where('lower(extern_uid) = ?', uid.downcase).last + identity && identity.user end end @@ -34,19 +35,21 @@ module Gitlab end def find_by_email - model.find_by(email: auth_hash.email) + ::User.find_by(email: auth_hash.email) end def update_user_attributes - gl_user.attributes = { - extern_uid: auth_hash.uid, - provider: auth_hash.provider, - email: auth_hash.email - } + gl_user.email = auth_hash.email + + # Build new identity only if we dont have have same one + gl_user.identities.find_or_initialize_by(provider: auth_hash.provider, + extern_uid: auth_hash.uid) + + gl_user end def changed? - gl_user.changed? + gl_user.changed? || gl_user.identities.any?(&:changed?) end def needs_blocking? diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb index 068c342398b..a1fd794aed2 100644 --- a/lib/gitlab/markdown.rb +++ b/lib/gitlab/markdown.rb @@ -92,7 +92,7 @@ module Gitlab allowed_tags = ActionView::Base.sanitized_allowed_tags sanitize text.html_safe, - attributes: allowed_attributes + %w(id class), + attributes: allowed_attributes + %w(id class style), tags: allowed_tags + %w(table tr td th) end @@ -128,6 +128,7 @@ module Gitlab (?<prefix>\W)? # Prefix ( # Reference @(?<user>#{NAME_STR}) # User name + |~(?<label>\d+) # Label ID |(?<issue>([A-Z\-]+-)\d+) # JIRA Issue ID |#{PROJ_STR}?\#(?<issue>([a-zA-Z\-]+-)?\d+) # Issue ID |#{PROJ_STR}?!(?<merge_request>\d+) # MR ID @@ -138,7 +139,7 @@ module Gitlab (?<suffix>\W)? # Suffix }x.freeze - TYPES = [:user, :issue, :merge_request, :snippet, :commit].freeze + TYPES = [:user, :issue, :label, :merge_request, :snippet, :commit].freeze def parse_references(text, project = @project) # parse reference links @@ -201,14 +202,34 @@ module Gitlab ) if identifier == "all" - link_to("@all", project_url(project), options) - elsif User.find_by(username: identifier) - link_to("@#{identifier}", user_url(identifier), options) + link_to("@all", namespace_project_url(project.namespace, project), options) + elsif namespace = Namespace.find_by(path: identifier) + url = + if namespace.type == "Group" + group_url(identifier) + else + user_url(identifier) + end + + link_to("@#{identifier}", url, options) + end + end + + def reference_label(identifier, project = @project, _ = nil) + if label = project.labels.find_by(id: identifier) + options = html_options.merge( + class: "gfm gfm-label #{html_options[:class]}" + ) + link_to( + render_colored_label(label), + namespace_project_issues_path(project.namespace, project, label_name: label.name), + options + ) end end def reference_issue(identifier, project = @project, prefix_text = nil) - if project.used_default_issues_tracker? || !external_issues_tracker_enabled? + if project.default_issues_tracker? if project.issue_exists? identifier url = url_for_issue(identifier, project) title = title_for_issue(identifier, project) @@ -220,10 +241,8 @@ module Gitlab link_to("#{prefix_text}##{identifier}", url, options) end else - config = Gitlab.config - external_issue_tracker = config.issues_tracker[project.issues_tracker] - if external_issue_tracker.present? - reference_external_issue(identifier, external_issue_tracker, project, + if project.external_issue_tracker.present? + reference_external_issue(identifier, project, prefix_text) end end @@ -236,7 +255,8 @@ module Gitlab title: "Merge Request: #{merge_request.title}", class: "gfm gfm-merge_request #{html_options[:class]}" ) - url = project_merge_request_url(project, merge_request) + url = namespace_project_merge_request_url(project.namespace, project, + merge_request) link_to("#{prefix_text}!#{identifier}", url, options) end end @@ -247,8 +267,11 @@ module Gitlab title: "Snippet: #{snippet.title}", class: "gfm gfm-snippet #{html_options[:class]}" ) - link_to("$#{identifier}", project_snippet_url(project, snippet), - options) + link_to( + "$#{identifier}", + namespace_project_snippet_url(project.namespace, project, snippet), + options + ) end end @@ -261,16 +284,16 @@ module Gitlab prefix_text = "#{prefix_text}@" if prefix_text link_to( "#{prefix_text}#{identifier}", - project_commit_url(project, commit), + namespace_project_commit_url(project.namespace, project, commit), options ) end end - def reference_external_issue(identifier, issue_tracker, project = @project, + def reference_external_issue(identifier, project = @project, prefix_text = nil) url = url_for_issue(identifier, project) - title = issue_tracker['title'] + title = project.external_issue_tracker.title options = html_options.merge( title: "Issue in #{title}", diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb new file mode 100644 index 00000000000..85ffa8aca68 --- /dev/null +++ b/lib/gitlab/middleware/static.rb @@ -0,0 +1,13 @@ +module Gitlab + module Middleware + class Static < ActionDispatch::Static + UPLOADS_REGEX = /\A\/uploads(\/|\z)/.freeze + + def call(env) + return @app.call(env) if env['PATH_INFO'] =~ UPLOADS_REGEX + + super + end + end + end +end diff --git a/lib/gitlab/oauth/user.rb b/lib/gitlab/oauth/user.rb index 47f62153a50..c023d275703 100644 --- a/lib/gitlab/oauth/user.rb +++ b/lib/gitlab/oauth/user.rb @@ -5,6 +5,8 @@ # module Gitlab module OAuth + class ForbiddenAction < StandardError; end + class User attr_accessor :auth_hash, :gl_user @@ -70,24 +72,25 @@ module Gitlab end def find_by_uid_and_provider - model.where(provider: auth_hash.provider, extern_uid: auth_hash.uid).last + identity = Identity.find_by(provider: auth_hash.provider, extern_uid: auth_hash.uid) + identity && identity.user end def build_new_user - model.new(user_attributes).tap do |user| - user.skip_confirmation! - end + user = ::User.new(user_attributes) + user.skip_confirmation! + user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) + user end def user_attributes { - extern_uid: auth_hash.uid, - provider: auth_hash.provider, - name: auth_hash.name, - username: auth_hash.username, - email: auth_hash.email, - password: auth_hash.password, - password_confirmation: auth_hash.password, + name: auth_hash.name, + username: ::User.clean_username(auth_hash.username), + email: auth_hash.email, + password: auth_hash.password, + password_confirmation: auth_hash.password, + password_automatically_set: true } end @@ -95,12 +98,8 @@ module Gitlab Gitlab::AppLogger end - def model - ::User - end - - def raise_unauthorized_to_create - raise StandardError.new("Unauthorized to create user, signup disabled for #{auth_hash.provider}") + def unauthorized_to_create + raise ForbiddenAction.new("Unauthorized to create user, signup disabled for #{auth_hash.provider}") end end end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index e2fbafb3899..fea4d2d55d2 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -21,6 +21,9 @@ module Gitlab @cmd_output = "" @cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + # We are not using stdin so we should close it, in case the command we + # are running waits for input. + stdin.close @cmd_output << stdout.read @cmd_output << stderr.read @cmd_status = wait_thr.value.exitstatus diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb new file mode 100644 index 00000000000..9aa5c8967a7 --- /dev/null +++ b/lib/gitlab/push_data_builder.rb @@ -0,0 +1,83 @@ +module Gitlab + class PushDataBuilder + class << self + # Produce a hash of post-receive data + # + # data = { + # before: String, + # after: String, + # ref: String, + # user_id: String, + # user_name: String, + # project_id: String, + # repository: { + # name: String, + # url: String, + # description: String, + # homepage: String, + # }, + # commits: Array, + # total_commits_count: Fixnum + # } + # + def build(project, user, oldrev, newrev, ref, commits = []) + # Total commits count + commits_count = commits.size + + # Get latest 20 commits ASC + commits_limited = commits.last(20) + + # Hash to be passed as post_receive_data + data = { + before: oldrev, + after: newrev, + ref: ref, + checkout_sha: checkout_sha(project.repository, newrev, ref), + user_id: user.id, + user_name: user.name, + project_id: project.id, + repository: { + name: project.name, + url: project.url_to_repo, + description: project.description, + homepage: project.web_url, + git_http_url: project.http_url_to_repo, + git_ssh_url: project.ssh_url_to_repo, + visibility_level: project.visibility_level + }, + commits: [], + total_commits_count: commits_count + } + + # For performance purposes maximum 20 latest commits + # will be passed as post receive hook data. + commits_limited.each do |commit| + data[:commits] << commit.hook_attrs(project) + end + + data + end + + # This method provide a sample data generated with + # existing project and commits to test web hooks + def build_sample(project, user) + commits = project.repository.commits(project.default_branch, nil, 3) + build(project, user, commits.last.id, commits.first.id, "refs/heads/#{project.default_branch}", commits) + end + + def checkout_sha(repository, newrev, ref) + if newrev != Gitlab::Git::BLANK_SHA && ref.start_with?('refs/tags/') + tag_name = Gitlab::Git.extract_ref_name(ref) + tag = repository.find_tag(tag_name) + + if tag + commit = repository.commit(tag.target) + commit.try(:sha) + end + else + newrev + end + end + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 99165950aef..7e5c991a222 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,12 +1,13 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor - attr_accessor :users, :issues, :merge_requests, :snippets, :commits + attr_accessor :users, :labels, :issues, :merge_requests, :snippets, :commits include Markdown def initialize - @users, @issues, @merge_requests, @snippets, @commits = [], [], [], [], [] + @users, @labels, @issues, @merge_requests, @snippets, @commits = + [], [], [], [], [], [] end def analyze(string, project) @@ -22,6 +23,12 @@ module Gitlab end.reject(&:nil?) end + def labels_for(project = nil) + labels.map do |entry| + project.labels.where(id: entry[:id]).first + end.reject(&:nil?) + end + def issues_for(project = nil) issues.map do |entry| if should_lookup?(project, entry[:project]) @@ -64,7 +71,7 @@ module Gitlab if entry_project.nil? false else - project.nil? || project.id == entry_project.id + project.nil? || entry_project.default_issues_tracker? end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 4b8038843b0..cf6e260f257 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -11,7 +11,7 @@ module Gitlab end def project_name_regex - /\A[a-zA-Z0-9_][a-zA-Z0-9_\-\. ]*\z/ + /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\. ]*\z/ end def project_regex_message @@ -67,8 +67,7 @@ module Gitlab def default_regex_message "can contain only letters, digits, '_', '-' and '.'. " \ - "It must start with letter, digit or '_', optionally preceeded by '.'. " \ - "It must not end in '.git'." + "Cannot start with '-' or end in '.git'" \ end def default_regex diff --git a/lib/gitlab/satellite/action.rb b/lib/gitlab/satellite/action.rb index be45cb5c98e..4890ccf21e6 100644 --- a/lib/gitlab/satellite/action.rb +++ b/lib/gitlab/satellite/action.rb @@ -44,7 +44,7 @@ module Gitlab end def default_options(options = {}) - {raise: true, timeout: true}.merge(options) + { raise: true, timeout: true }.merge(options) end def handle_exception(exception) diff --git a/lib/gitlab/satellite/files/delete_file_action.rb b/lib/gitlab/satellite/files/delete_file_action.rb index 30462999aa3..0d37b9dea85 100644 --- a/lib/gitlab/satellite/files/delete_file_action.rb +++ b/lib/gitlab/satellite/files/delete_file_action.rb @@ -13,7 +13,7 @@ module Gitlab prepare_satellite!(repo) # create target branch in satellite at the corresponding commit from bare repo - repo.git.checkout({raise: true, timeout: true, b: true}, ref, "origin/#{ref}") + repo.git.checkout({ raise: true, timeout: true, b: true }, ref, "origin/#{ref}") # update the file in the satellite's working dir file_path_in_satellite = File.join(repo.working_dir, file_path) @@ -36,7 +36,7 @@ module Gitlab # push commit back to bare repo # will raise CommandFailed when push fails - repo.git.push({raise: true, timeout: true}, :origin, ref) + repo.git.push({ raise: true, timeout: true }, :origin, ref) # everything worked true diff --git a/lib/gitlab/satellite/files/edit_file_action.rb b/lib/gitlab/satellite/files/edit_file_action.rb index cbdf70f7d12..3cb9c0b5ecb 100644 --- a/lib/gitlab/satellite/files/edit_file_action.rb +++ b/lib/gitlab/satellite/files/edit_file_action.rb @@ -10,12 +10,16 @@ module Gitlab # Returns false if committing the change fails # Returns false if pushing from the satellite to bare repo failed or was rejected # Returns true otherwise - def commit!(content, commit_message, encoding) + def commit!(content, commit_message, encoding, new_branch = nil) in_locked_and_timed_satellite do |repo| prepare_satellite!(repo) # create target branch in satellite at the corresponding commit from bare repo - repo.git.checkout({raise: true, timeout: true, b: true}, ref, "origin/#{ref}") + begin + repo.git.checkout({ raise: true, timeout: true, b: true }, ref, "origin/#{ref}") + rescue Grit::Git::CommandFailed => ex + log_and_raise(CheckoutFailed, ex.message) + end # update the file in the satellite's working dir file_path_in_satellite = File.join(repo.working_dir, file_path) @@ -31,19 +35,33 @@ module Gitlab # commit the changes # will raise CommandFailed when commit fails - repo.git.commit(raise: true, timeout: true, a: true, m: commit_message) + begin + repo.git.commit(raise: true, timeout: true, a: true, m: commit_message) + rescue Grit::Git::CommandFailed => ex + log_and_raise(CommitFailed, ex.message) + end + target_branch = new_branch.present? ? "#{ref}:#{new_branch}" : ref + # push commit back to bare repo # will raise CommandFailed when push fails - repo.git.push({raise: true, timeout: true}, :origin, ref) + begin + repo.git.push({ raise: true, timeout: true }, :origin, target_branch) + rescue Grit::Git::CommandFailed => ex + log_and_raise(PushFailed, ex.message) + end # everything worked true end - rescue Grit::Git::CommandFailed => ex - Gitlab::GitLogger.error(ex.message) - false + end + + private + + def log_and_raise(errorClass, message) + Gitlab::GitLogger.error(message) + raise(errorClass, message) end end end diff --git a/lib/gitlab/satellite/files/new_file_action.rb b/lib/gitlab/satellite/files/new_file_action.rb index 15e9b7a6f77..724dfa0d042 100644 --- a/lib/gitlab/satellite/files/new_file_action.rb +++ b/lib/gitlab/satellite/files/new_file_action.rb @@ -9,12 +9,19 @@ module Gitlab # Returns false if committing the change fails # Returns false if pushing from the satellite to bare repo failed or was rejected # Returns true otherwise - def commit!(content, commit_message, encoding) + def commit!(content, commit_message, encoding, new_branch = nil) in_locked_and_timed_satellite do |repo| prepare_satellite!(repo) # create target branch in satellite at the corresponding commit from bare repo - repo.git.checkout({raise: true, timeout: true, b: true}, ref, "origin/#{ref}") + current_ref = + if @project.empty_repo? + # skip this step if we want to add first file to empty repo + Satellite::PARKING_BRANCH + else + repo.git.checkout({ raise: true, timeout: true, b: true }, ref, "origin/#{ref}") + ref + end file_path_in_satellite = File.join(repo.working_dir, file_path) dir_name_in_satellite = File.dirname(file_path_in_satellite) @@ -38,10 +45,15 @@ module Gitlab # will raise CommandFailed when commit fails repo.git.commit(raise: true, timeout: true, a: true, m: commit_message) + target_branch = if new_branch.present? && !@project.empty_repo? + "#{ref}:#{new_branch}" + else + "#{current_ref}:#{ref}" + end # push commit back to bare repo # will raise CommandFailed when push fails - repo.git.push({raise: true, timeout: true}, :origin, ref) + repo.git.push({ raise: true, timeout: true }, :origin, target_branch) # everything worked true diff --git a/lib/gitlab/satellite/merge_action.rb b/lib/gitlab/satellite/merge_action.rb index e9141f735aa..25122666f5e 100644 --- a/lib/gitlab/satellite/merge_action.rb +++ b/lib/gitlab/satellite/merge_action.rb @@ -86,7 +86,7 @@ module Gitlab in_locked_and_timed_satellite do |merge_repo| prepare_satellite!(merge_repo) update_satellite_source_and_target!(merge_repo) - patch = merge_repo.git.format_patch(default_options({stdout: true}), "origin/#{merge_request.target_branch}..source/#{merge_request.source_branch}") + patch = merge_repo.git.format_patch(default_options({ stdout: true }), "origin/#{merge_request.target_branch}..source/#{merge_request.source_branch}") end rescue Grit::Git::CommandFailed => ex handle_exception(ex) @@ -128,7 +128,7 @@ module Gitlab # merge the source branch into the satellite # will raise CommandFailed when merge fails - repo.git.merge(default_options({no_ff: true}), "-m#{message}", "source/#{merge_request.source_branch}") + repo.git.merge(default_options({ no_ff: true }), "-m#{message}", "source/#{merge_request.source_branch}") rescue Grit::Git::CommandFailed => ex handle_exception(ex) end @@ -137,7 +137,7 @@ module Gitlab def update_satellite_source_and_target!(repo) repo.remote_add('source', merge_request.source_project.repository.path_to_repo) repo.remote_fetch('source') - repo.git.checkout(default_options({b: true}), merge_request.target_branch, "origin/#{merge_request.target_branch}") + repo.git.checkout(default_options({ b: true }), merge_request.target_branch, "origin/#{merge_request.target_branch}") rescue Grit::Git::CommandFailed => ex handle_exception(ex) end diff --git a/lib/gitlab/satellite/satellite.rb b/lib/gitlab/satellite/satellite.rb index 1de84309d15..70125d539da 100644 --- a/lib/gitlab/satellite/satellite.rb +++ b/lib/gitlab/satellite/satellite.rb @@ -1,5 +1,9 @@ module Gitlab module Satellite + class CheckoutFailed < StandardError; end + class CommitFailed < StandardError; end + class PushFailed < StandardError; end + class Satellite include Gitlab::Popen @@ -98,13 +102,13 @@ module Gitlab if heads.include? PARKING_BRANCH repo.git.checkout({}, PARKING_BRANCH) else - repo.git.checkout(default_options({b: true}), PARKING_BRANCH) + repo.git.checkout(default_options({ b: true }), PARKING_BRANCH) end # remove the parking branch from the list of heads ... heads.delete(PARKING_BRANCH) # ... and delete all others - heads.each { |head| repo.git.branch(default_options({D: true}), head) } + heads.each { |head| repo.git.branch(default_options({ D: true }), head) } end # Deletes all remotes except origin @@ -126,7 +130,7 @@ module Gitlab end def default_options(options = {}) - {raise: true, timeout: true}.merge(options) + { raise: true, timeout: true }.merge(options) end # Create directory for storing diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb new file mode 100644 index 00000000000..0f2db50e98c --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -0,0 +1,53 @@ +module Gitlab + module SidekiqMiddleware + class MemoryKiller + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # Create a mutex used to ensure there will be only one thread waiting to + # shut Sidekiq down + MUTEX = Mutex.new + + def call(worker, job, queue) + yield + current_rss = get_rss + + return unless MAX_RSS > 0 && current_rss > MAX_RSS + + Thread.new do + # Return if another thread is already waiting to shut Sidekiq down + return unless MUTEX.try_lock + + Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ + "#{MAX_RSS}" + Sidekiq.logger.warn "spawned thread that will shut down PID "\ + "#{Process.pid} in #{GRACE_TIME} seconds" + sleep(GRACE_TIME) + + Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" + Process.kill('SIGUSR1', Process.pid) + + Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ + "SIGTERM to PID #{Process.pid}" + sleep(SHUTDOWN_WAIT) + + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" + Process.kill('SIGTERM', Process.pid) + end + end + + private + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid})) + return 0 unless status.zero? + + output.to_i + end + end + end +end diff --git a/lib/gitlab/theme.rb b/lib/gitlab/theme.rb index b7c50cb734d..a7c83a880f6 100644 --- a/lib/gitlab/theme.rb +++ b/lib/gitlab/theme.rb @@ -19,5 +19,19 @@ module Gitlab return themes[id] end + + def self.type_css_class_by_id(id) + types = { + BASIC => 'light_theme', + MARS => 'dark_theme', + MODERN => 'dark_theme', + GRAY => 'dark_theme', + COLOR => 'dark_theme' + } + + id ||= Gitlab.config.gitlab.default_theme + + types[id] + end end end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index 74b049b5143..0570c2fbeb5 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -62,7 +62,7 @@ module Gitlab end def env - {'RAILS_ENV' => 'production'} + { 'RAILS_ENV' => 'production' } end def upgrade diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 877488d8471..e7153cc3225 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -17,9 +17,10 @@ module Gitlab def issue_url(id) issue = Issue.find(id) - project_issue_url(id: issue.iid, - project_id: issue.project, - host: Gitlab.config.gitlab['url']) + namespace_project_issue_url(namespace_id: issue.project.namespace, + id: issue.iid, + project_id: issue.project, + host: Gitlab.config.gitlab['url']) end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb new file mode 100644 index 00000000000..bd184c27187 --- /dev/null +++ b/lib/gitlab/utils.rb @@ -0,0 +1,13 @@ +module Gitlab + module Utils + extend self + + # Run system command without outputting to stdout. + # + # @param cmd [Array<String>] + # @return [Boolean] + def system_silent(cmd) + Popen::popen(cmd).last.zero? + end + end +end diff --git a/lib/redcarpet/render/gitlab_html.rb b/lib/redcarpet/render/gitlab_html.rb index 54d740908d5..714261f815c 100644 --- a/lib/redcarpet/render/gitlab_html.rb +++ b/lib/redcarpet/render/gitlab_html.rb @@ -21,23 +21,22 @@ class Redcarpet::Render::GitlabHTML < Redcarpet::Render::HTML text.gsub("'", "’") end + # Stolen from Rugments::Plugins::Redcarpet as this module is not required + # from Rugments's gem root. def block_code(code, language) - # New lines are placed to fix an rendering issue - # with code wrapped inside <h1> tag for next case: - # - # # Title kinda h1 - # - # ruby code here - # - <<-HTML + lexer = Rugments::Lexer.find_fancy(language, code) || Rugments::Lexers::PlainText -<div class="highlighted-data #{h.user_color_scheme_class}"> - <div class="highlight"> - <pre><code class="#{language}">#{h.send(:html_escape, code)}</code></pre> - </div> -</div> + # XXX HACK: Redcarpet strips hard tabs out of code blocks, + # so we assume you're not using leading spaces that aren't tabs, + # and just replace them here. + if lexer.tag == 'make' + code.gsub! /^ /, "\t" + end - HTML + formatter = Rugments::Formatters::HTML.new( + cssclass: "code highlight white #{lexer.tag}" + ) + formatter.format(lexer.lex(code)) end def link(link, title, content) diff --git a/lib/repository_cache.rb b/lib/repository_cache.rb new file mode 100644 index 00000000000..fa016a170cd --- /dev/null +++ b/lib/repository_cache.rb @@ -0,0 +1,21 @@ +# Interface to the Redis-backed cache store used by the Repository model +class RepositoryCache + attr_reader :namespace, :backend + + def initialize(namespace, backend = Rails.cache) + @namespace = namespace + @backend = backend + end + + def cache_key(type) + "#{type}:#{namespace}" + end + + def expire(key) + backend.delete(cache_key(key)) + end + + def fetch(key, &block) + backend.fetch(cache_key(key), &block) + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 49a68c62293..fd5b2664786 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -1,5 +1,5 @@ ## GitLab -## Maintainer: @randx +## Contributors: randx, yin8086, sashkab, orkoden, axilleas, bbodenmiller, DouweM ## ## Lines starting with two hashes (##) are comments with information. ## Lines starting with one hash (#) are configuration parameters that can be uncommented. @@ -15,7 +15,7 @@ ## - installing an old version of Nginx with the chunkin module [2] compiled in, or ## - using a newer version of Nginx. ## -## At the time of writing we do not know if either of these theoretical solutions works. +## At the time of writing we do not know if either of these theoretical solutions works. ## As a workaround users can use Git over SSH to push large files. ## ## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99 @@ -26,6 +26,7 @@ ## configuration ## ################################### ## +## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; @@ -33,7 +34,8 @@ upstream gitlab { ## Normal HTTP host server { - listen *:80 default_server; + listen 0.0.0.0:80 default_server; + listen [::]:80 default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice root /home/git/gitlab/public; @@ -42,6 +44,8 @@ server { ## Or if you want to accept large git objects over http client_max_body_size 20m; + ## See app/controllers/application_controller.rb for headers set + ## Individual nginx logs for this GitLab vhost access_log /var/log/nginx/gitlab_access.log; error_log /var/log/nginx/gitlab_error.log; @@ -52,6 +56,37 @@ server { try_files $uri $uri/index.html $uri.html @gitlab; } + ## We route uploads through GitLab to prevent XSS and enforce access control. + location /uploads/ { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + # gzip off; + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frame-Options SAMEORIGIN; + + proxy_pass http://gitlab; + } + + ## If ``go get`` detected, return go-import meta tag. + ## This works for public and for private repositories. + ## See also http://golang.org/cmd/go/#hdr-Remote_import_paths + if ($http_user_agent ~* "Go") { + return 200 " + <!DOCTYPE html> + <head><meta content='$host$uri git $scheme://$host$uri.git' name='go-import'></head> + </html>"; + } + ## If a file, which is not found in the root folder is requested, ## then the proxy passes the request to the upsteam (gitlab unicorn). location @gitlab { diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index cbb198086b5..a9699bac611 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -1,5 +1,5 @@ ## GitLab -## Contributors: randx, yin8086, sashkab, orkoden, axilleas +## Contributors: randx, yin8086, sashkab, orkoden, axilleas, bbodenmiller, DouweM ## ## Modified from nginx http version ## Modified from http://blog.phusion.nl/2012/04/21/tutorial-setting-up-gitlab-on-debian-6/ @@ -26,9 +26,8 @@ ## [1] https://github.com/agentzh/chunkin-nginx-module#status ## [2] https://github.com/agentzh/chunkin-nginx-module ## -## ################################### -## SSL configuration ## +## configuration ## ################################### ## ## See installation.md#using-https for additional HTTPS configuration details. @@ -37,22 +36,24 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -## Normal HTTP host +## Redirects all HTTP traffic to the HTTPS host server { - listen *:80 default_server; + listen 0.0.0.0:80; + listen [::]:80 ipv6only=on default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice - - ## Redirects all traffic to the HTTPS host - root /nowhere; ## root doesn't have to be a valid path since we are redirecting - rewrite ^ https://$server_name$request_uri? permanent; + return 301 https://$server_name$request_uri; + access_log /var/log/nginx/gitlab_access.log; + error_log /var/log/nginx/gitlab_error.log; } + ## HTTPS host server { - listen 443 ssl; + listen 0.0.0.0:443 ssl; + listen [::]:443 ipv6only=on ssl default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com - server_tokens off; + server_tokens off; ## Don't show the nginx version number, a security best practice root /home/git/gitlab/public; ## Increase this if you want to upload large attachments @@ -70,12 +71,9 @@ server { ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_prefer_server_ciphers on; ssl_session_cache shared:SSL:10m; + ssl_session_timeout 5m; - ## [WARNING] The following header states that the browser should only communicate - ## with your server over a secure connection for the next 24 months. - add_header Strict-Transport-Security max-age=63072000; - add_header X-Frame-Options SAMEORIGIN; - add_header X-Content-Type-Options nosniff; + ## See app/controllers/application_controller.rb for headers set ## [Optional] If your certficate has OCSP, enable OCSP stapling to reduce the overhead and latency of running SSL. ## Replace with your ssl_trusted_certificate. For more info see: @@ -103,6 +101,38 @@ server { try_files $uri $uri/index.html $uri.html @gitlab; } + ## We route uploads through GitLab to prevent XSS and enforce access control. + location /uploads/ { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + gzip off; + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Ssl on; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frame-Options SAMEORIGIN; + + proxy_pass http://gitlab; + } + + ## If ``go get`` detected, return go-import meta tag. + ## This works for public and for private repositories. + ## See also http://golang.org/cmd/go/#hdr-Remote_import_paths + if ($http_user_agent ~* "Go") { + return 200 " + <!DOCTYPE html> + <head><meta content='$host$uri git $scheme://$host$uri.git' name='go-import'></head> + </html>"; + } + ## If a file, which is not found in the root folder is requested, ## then the proxy passes the request to the upsteam (gitlab unicorn). location @gitlab { diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 2eff1260b61..0230fbb010b 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -6,6 +6,7 @@ namespace :gitlab do desc "GITLAB | Create a backup of the GitLab system" task create: :environment do warn_user_is_not_gitlab + configure_cron_mode Rake::Task["gitlab:backup:db:create"].invoke Rake::Task["gitlab:backup:repo:create"].invoke @@ -21,6 +22,7 @@ namespace :gitlab do desc "GITLAB | Restore a previously created backup" task restore: :environment do warn_user_is_not_gitlab + configure_cron_mode backup = Backup::Manager.new backup.unpack @@ -35,43 +37,54 @@ namespace :gitlab do namespace :repo do task create: :environment do - puts "Dumping repositories ...".blue + $progress.puts "Dumping repositories ...".blue Backup::Repository.new.dump - puts "done".green + $progress.puts "done".green end task restore: :environment do - puts "Restoring repositories ...".blue + $progress.puts "Restoring repositories ...".blue Backup::Repository.new.restore - puts "done".green + $progress.puts "done".green end end namespace :db do task create: :environment do - puts "Dumping database ... ".blue + $progress.puts "Dumping database ... ".blue Backup::Database.new.dump - puts "done".green + $progress.puts "done".green end task restore: :environment do - puts "Restoring database ... ".blue + $progress.puts "Restoring database ... ".blue Backup::Database.new.restore - puts "done".green + $progress.puts "done".green end end namespace :uploads do task create: :environment do - puts "Dumping uploads ... ".blue + $progress.puts "Dumping uploads ... ".blue Backup::Uploads.new.dump - puts "done".green + $progress.puts "done".green end task restore: :environment do - puts "Restoring uploads ... ".blue + $progress.puts "Restoring uploads ... ".blue Backup::Uploads.new.restore - puts "done".green + $progress.puts "done".green + end + end + + def configure_cron_mode + if ENV['CRON'] + # We need an object we can say 'puts' and 'print' to; let's use a + # StringIO. + require 'stringio' + $progress = StringIO.new + else + $progress = $stdout end end end # namespace end: backup diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 56e8ff44988..43115915de1 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -574,24 +574,16 @@ namespace :gitlab do Gitlab::Shell.new.version end - def required_gitlab_shell_version - File.read(File.join(Rails.root, "GITLAB_SHELL_VERSION")).strip - end - def gitlab_shell_major_version - required_gitlab_shell_version.split(".")[0].to_i + Gitlab::Shell.version_required.split('.')[0].to_i end def gitlab_shell_minor_version - required_gitlab_shell_version.split(".")[1].to_i + Gitlab::Shell.version_required.split('.')[1].to_i end def gitlab_shell_patch_version - required_gitlab_shell_version.split(".")[2].to_i - end - - def has_gitlab_shell3? - gitlab_shell_version.try(:start_with?, "v3.") + Gitlab::Shell.version_required.split('.')[2].to_i end end @@ -794,14 +786,14 @@ namespace :gitlab do end def sanitized_message(project) - if sanitize + if should_sanitize? "#{project.namespace_id.to_s.yellow}/#{project.id.to_s.yellow} ... " else "#{project.name_with_namespace.yellow} ... " end end - def sanitize + def should_sanitize? if ENV['SANITIZE'] == "true" true else diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 63dcdc52370..189ad6090a4 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -92,11 +92,11 @@ namespace :gitlab do User.ldap.each do |ldap_user| print "#{ldap_user.name} (#{ldap_user.extern_uid}) ..." - if Gitlab::LDAP::Access.open { |access| access.allowed?(ldap_user) } + if Gitlab::LDAP::Access.allowed?(ldap_user) puts " [OK]".green else if block_flag - ldap_user.block! + ldap_user.block! unless ldap_user.blocked? puts " [BLOCKED]".red else puts " [NOT IN LDAP]".yellow diff --git a/lib/tasks/gitlab/db/drop_all_postgres_sequences.rake b/lib/tasks/gitlab/db/drop_all_postgres_sequences.rake new file mode 100644 index 00000000000..e9cf0a9b5e8 --- /dev/null +++ b/lib/tasks/gitlab/db/drop_all_postgres_sequences.rake @@ -0,0 +1,10 @@ +namespace :gitlab do + namespace :db do + task drop_all_postgres_sequences: :environment do + connection = ActiveRecord::Base.connection + connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence| + connection.execute("DROP SEQUENCE #{sequence['relname']}") + end + end + end +end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 3c693546c09..20abb2fa500 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -25,7 +25,7 @@ namespace :gitlab do puts "Processing #{repo_path}".yellow - if path =~ /\.wiki\Z/ + if path.end_with?('.wiki') puts " * Skipping wiki repo" next end @@ -66,6 +66,7 @@ namespace :gitlab do puts " * Created #{project.name} (#{repo_path})".green else puts " * Failed trying to create #{project.name} (#{repo_path})".red + puts " Validation Errors: #{project.errors.messages}".red end end end diff --git a/lib/tasks/gitlab/mail_google_schema_whitelisting.rake b/lib/tasks/gitlab/mail_google_schema_whitelisting.rake new file mode 100644 index 00000000000..102c6ae55d5 --- /dev/null +++ b/lib/tasks/gitlab/mail_google_schema_whitelisting.rake @@ -0,0 +1,73 @@ +require "#{Rails.root}/app/helpers/emails_helper" +require 'action_view/helpers' +extend ActionView::Helpers + +include ActionView::Context +include EmailsHelper + +namespace :gitlab do + desc "Email google whitelisting email with example email for actions in inbox" + task mail_google_schema_whitelisting: :environment do + subject = "Rails | Implemented feature" + url = "#{Gitlab.config.gitlab.url}/base/rails-project/issues/#{rand(1..100)}#note_#{rand(10..1000)}" + schema = email_action(url) + body = email_template(schema, url) + mail = Notify.test_email("schema.whitelisting+sample@gmail.com", subject, body.html_safe) + if send_now + mail.deliver + else + puts "WOULD SEND:" + end + puts mail + end + + def email_template(schema, url) + "<html lang='en'> + <head> + <meta content='text/html; charset=utf-8' http-equiv='Content-Type'> + <title> + GitLab + </title> + </meta> + </head> + <style> + img { + max-width: 100%; + height: auto; + } + p.details { + font-style:italic; + color:#777 + } + .footer p { + font-size:small; + color:#777 + } + </style> + <body> + <div class='content'> + <div> + <p>I like it :+1: </p> + </div> + </div> + + <div class='footer' style='margin-top: 10px;'> + <p> + <br> + <a href=\"#{url}\">View it on GitLab</a> + You're receiving this notification because you are a member of the Base / Rails Project project team. + #{schema} + </p> + </div> + </body> + </html>" + end + + def send_now + if ENV['SEND'] == "true" + true + else + false + end + end +end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 55f338add6a..9af93300e08 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -4,7 +4,7 @@ namespace :gitlab do task :install, [:tag, :repo] => :environment do |t, args| warn_user_is_not_gitlab - default_version = File.read(File.join(Rails.root, "GITLAB_SHELL_VERSION")).strip + default_version = Gitlab::Shell.version_required args.with_defaults(tag: 'v' + default_version, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git") user = Gitlab.config.gitlab.user @@ -17,15 +17,19 @@ namespace :gitlab do # Clone if needed unless File.directory?(target_dir) - sh(*%W(git clone #{args.repo} #{target_dir})) + system(*%W(git clone -- #{args.repo} #{target_dir})) end # Make sure we're on the right tag Dir.chdir(target_dir) do # First try to checkout without fetching # to avoid stalling tests if the Internet is down. - reset = "git reset --hard $(git describe #{args.tag} || git describe origin/#{args.tag})" - sh "#{reset} || git fetch origin && #{reset}" + reseted = reset_to_commit(args) + + unless reseted + system(*%W(git fetch origin)) + reset_to_commit(args) + end config = { user: user, @@ -54,7 +58,7 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - sh "bin/install" + system(*%W(bin/install)) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -76,7 +80,7 @@ namespace :gitlab do desc "GITLAB | Build missing projects" task build_missing_projects: :environment do Project.find_each(batch_size: 1000) do |project| - path_to_repo = File.join(Gitlab.config.gitlab_shell.repos_path, "#{project.path_with_namespace}.git") + path_to_repo = project.repository.path_to_repo if File.exists?(path_to_repo) print '-' else @@ -118,5 +122,16 @@ namespace :gitlab do puts "Quitting...".red exit 1 end + + def reset_to_commit(args) + tag, status = Gitlab::Popen.popen(%W(git describe -- #{args.tag})) + + unless status.zero? + tag, status = Gitlab::Popen.popen(%W(git describe -- origin/#{args.tag})) + end + + tag = tag.strip + system(*%W(git reset --hard #{tag})) + end end diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake index c01b00bd1c0..b4076f8238f 100644 --- a/lib/tasks/gitlab/test.rake +++ b/lib/tasks/gitlab/test.rake @@ -2,6 +2,7 @@ namespace :gitlab do desc "GITLAB | Run all tests" task :test do cmds = [ + %W(rake rubocop), %W(rake spinach), %W(rake spec), %W(rake jasmine:ci) diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake new file mode 100644 index 00000000000..ddfaf5d51f2 --- /dev/null +++ b/lib/tasks/rubocop.rake @@ -0,0 +1,4 @@ +unless Rails.env.production? + require 'rubocop/rake_task' + RuboCop::RakeTask.new +end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index 507b315759d..4aefc18ce14 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -2,9 +2,15 @@ Rake::Task["spinach"].clear if Rake::Task.task_defined?('spinach') desc "GITLAB | Run spinach" task :spinach do + tags = if ENV['SEMAPHORE'] + '~@tricky' + else + '~@semaphore' + end + cmds = [ %W(rake gitlab:setup), - %W(spinach), + %W(spinach --tags #{tags}), ] run_commands(cmds) end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index 583f4a876da..3ea9290a814 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -9,5 +9,5 @@ unless Rails.env.production? require 'coveralls/rake/task' Coveralls::RakeTask.new desc "GITLAB | Run all tests on CI with simplecov" - task :test_ci => [:spinach, :spec, 'coveralls:push'] + task :test_ci => [:rubocop, :spinach, :spec, 'coveralls:push'] end |