diff options
Diffstat (limited to 'lib')
52 files changed, 1191 insertions, 437 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index c4c9f166db1..283f7642f67 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -38,5 +38,8 @@ module API mount ProjectSnippets mount DeployKeys mount ProjectHooks + mount Services + mount Files + mount Namespaces end end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 218b3d8eee2..7f5a125038c 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -5,16 +5,6 @@ module API before { authorize_admin_project } resource :projects do - helpers do - def handle_project_member_errors(errors) - if errors[:project_access].any? - error!(errors[:project_access], 422) - end - not_found! - end - end - - # Get a specific project's keys # # Example Request: @@ -48,14 +38,14 @@ module API attrs[:key].strip! # check if key already exist in project - key = user_project.deploy_keys.find_by_key(attrs[:key]) + key = user_project.deploy_keys.find_by(key: attrs[:key]) if key present key, with: Entities::SSHKey return end # Check for available deploy keys in other projects - key = current_user.accessible_deploy_keys.find_by_key(attrs[:key]) + key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) if key user_project.deploy_keys << key present key, with: Entities::SSHKey diff --git a/lib/api/entities.rb b/lib/api/entities.rb index b4771eecc7f..8f54d0d4d84 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,7 +1,7 @@ module API module Entities class User < Grape::Entity - expose :id, :username, :email, :name, :bio, :skype, :linkedin, :twitter, + expose :id, :username, :email, :name, :bio, :skype, :linkedin, :twitter, :website_url, :theme_id, :color_scheme_id, :state, :created_at, :extern_uid, :provider expose :is_admin?, as: :is_admin expose :can_create_group?, as: :can_create_group @@ -24,6 +24,10 @@ module API expose :id, :url, :created_at end + class ProjectHook < Hook + expose :project_id, :push_events, :issues_events, :merge_requests_events + end + class ForkedFromProject < Grape::Entity expose :id expose :name, :name_with_namespace @@ -31,30 +35,32 @@ module API end class Project < Grape::Entity - expose :id, :description, :default_branch, :public, :ssh_url_to_repo, :http_url_to_repo, :web_url + expose :id, :description, :default_branch + expose :public?, as: :public + expose :visibility_level, :ssh_url_to_repo, :http_url_to_repo, :web_url expose :owner, using: Entities::UserBasic expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wall_enabled, :wiki_enabled, :snippets_enabled, :created_at, :last_activity_at, :public + expose :issues_enabled, :merge_requests_enabled, :wall_enabled, :wiki_enabled, :snippets_enabled, :created_at, :last_activity_at expose :namespace expose :forked_from_project, using: Entities::ForkedFromProject, :if => lambda{ | project, options | project.forked? } end class ProjectMember < UserBasic expose :project_access, as: :access_level do |user, options| - options[:project].users_projects.find_by_user_id(user.id).project_access + options[:project].users_projects.find_by(user_id: user.id).project_access end end class TeamMember < UserBasic expose :permission, as: :access_level do |user, options| - options[:user_team].user_team_user_relationships.find_by_user_id(user.id).permission + options[:user_team].user_team_user_relationships.find_by(user_id: user.id).permission end end class TeamProject < Project expose :greatest_access, as: :greatest_access_level do |project, options| - options[:user_team].user_team_project_relationships.find_by_project_id(project.id).greatest_access + options[:user_team].user_team_project_relationships.find_by(project_id: project.id).greatest_access end end @@ -68,12 +74,21 @@ module API class GroupMember < UserBasic expose :group_access, as: :access_level do |user, options| - options[:group].users_groups.find_by_user_id(user.id).group_access + options[:group].users_groups.find_by(user_id: user.id).group_access end end class RepoObject < Grape::Entity - expose :name, :commit + expose :name + + expose :commit do |repo_obj, options| + if repo_obj.respond_to?(:commit) + repo_obj.commit + elsif options[:project] + options[:project].repository.commit(repo_obj.target) + end + end + expose :protected do |repo, options| if options[:project] options[:project].protected_branch? repo.name @@ -81,25 +96,40 @@ module API end end + class RepoTreeObject < Grape::Entity + expose :id, :name, :type + + expose :mode do |obj, options| + filemode = obj.mode.to_s(8) + filemode = "0" + filemode if filemode.length < 6 + filemode + end + end + class RepoCommit < Grape::Entity expose :id, :short_id, :title, :author_name, :author_email, :created_at end + class RepoCommitDetail < RepoCommit + expose :parent_ids, :committed_date, :authored_date + end + class ProjectSnippet < Grape::Entity expose :id, :title, :file_name expose :author, using: Entities::UserBasic expose :expires_at, :updated_at, :created_at end - class Milestone < Grape::Entity - expose :id - expose (:project_id) {|milestone| milestone.project.id} + class ProjectEntity < Grape::Entity + expose :id, :iid + expose (:project_id) { |entity| entity.project.id } + end + + class Milestone < ProjectEntity expose :title, :description, :due_date, :state, :updated_at, :created_at end - class Issue < Grape::Entity - expose :id - expose (:project_id) {|issue| issue.project.id} + class Issue < ProjectEntity expose :title, :description expose :label_list, as: :labels expose :milestone, using: Entities::Milestone @@ -107,14 +137,14 @@ module API expose :state, :updated_at, :created_at end - class SSHKey < Grape::Entity - expose :id, :title, :key, :created_at + class MergeRequest < ProjectEntity + expose :target_branch, :source_branch, :title, :state, :upvotes, :downvotes + expose :author, :assignee, using: Entities::UserBasic + expose :source_project_id, :target_project_id end - class MergeRequest < Grape::Entity - expose :id, :target_branch, :source_branch, :title, :state, :upvotes, :downvotes - expose :target_project_id, as: :project_id - expose :author, :assignee, using: Entities::UserBasic + class SSHKey < Grape::Entity + expose :id, :title, :key, :created_at end class Note < Grape::Entity @@ -135,5 +165,9 @@ module API expose :target_id, :target_type, :author_id expose :data, :target_title end + + class Namespace < Grape::Entity + expose :id, :path, :kind + end end end diff --git a/lib/api/files.rb b/lib/api/files.rb new file mode 100644 index 00000000000..213604915a6 --- /dev/null +++ b/lib/api/files.rb @@ -0,0 +1,99 @@ +module API + # Projects API + class Files < Grape::API + before { authenticate! } + before { authorize! :push_code, user_project } + + resource :projects do + # Create new file in repository + # + # Parameters: + # file_path (optional) - The path to new file. Ex. lib/class.rb + # branch_name (required) - The name of branch + # content (required) - File content + # commit_message (required) - Commit message + # + # Example Request: + # POST /projects/:id/repository/files + # + post ":id/repository/files" do + required_attributes! [:file_path, :branch_name, :content, :commit_message] + attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] + branch_name = attrs.delete(:branch_name) + file_path = attrs.delete(:file_path) + result = ::Files::CreateService.new(user_project, current_user, attrs, branch_name, file_path).execute + + if result[:status] == :success + status(201) + + { + file_path: file_path, + branch_name: branch_name + } + else + render_api_error!(result[:error], 400) + end + end + + # Update existing file in repository + # + # Parameters: + # file_path (optional) - The path to file. Ex. lib/class.rb + # branch_name (required) - The name of branch + # content (required) - File content + # commit_message (required) - Commit message + # + # Example Request: + # PUT /projects/:id/repository/files + # + put ":id/repository/files" do + required_attributes! [:file_path, :branch_name, :content, :commit_message] + attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] + branch_name = attrs.delete(:branch_name) + file_path = attrs.delete(:file_path) + result = ::Files::UpdateService.new(user_project, current_user, attrs, branch_name, file_path).execute + + if result[:status] == :success + status(200) + + { + file_path: file_path, + branch_name: branch_name + } + else + render_api_error!(result[:error], 400) + end + end + + # Delete existing file in repository + # + # Parameters: + # file_path (optional) - The path to file. Ex. lib/class.rb + # branch_name (required) - The name of branch + # content (required) - File content + # commit_message (required) - Commit message + # + # Example Request: + # DELETE /projects/:id/repository/files + # + delete ":id/repository/files" do + required_attributes! [:file_path, :branch_name, :commit_message] + attrs = attributes_for_keys [:file_path, :branch_name, :commit_message] + branch_name = attrs.delete(:branch_name) + file_path = attrs.delete(:file_path) + result = ::Files::DeleteService.new(user_project, current_user, attrs, branch_name, file_path).execute + + if result[:status] == :success + status(200) + + { + file_path: file_path, + branch_name: branch_name + } + else + render_api_error!(result[:error], 400) + end + end + end + end +end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 265417fd6bc..03f027706de 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -66,7 +66,6 @@ module API present group, with: Entities::GroupDetail end - # Remove group # # Parameters: @@ -122,11 +121,11 @@ module API render_api_error!("Wrong access level", 422) end group = find_group(params[:id]) - if group.users_groups.find_by_user_id(params[:user_id]) + if group.users_groups.find_by(user_id: params[:user_id]) render_api_error!("Already exists", 409) end group.add_users([params[:user_id]], params[:access_level]) - member = group.users_groups.find_by_user_id(params[:user_id]) + member = group.users_groups.find_by(user_id: params[:user_id]) present member.user, with: Entities::GroupMember, group: group end @@ -140,7 +139,7 @@ module API # DELETE /groups/:id/members/:user_id delete ":id/members/:user_id" do group = find_group(params[:id]) - member = group.users_groups.find_by_user_id(params[:user_id]) + member = group.users_groups.find_by(user_id: params[:user_id]) if member.nil? render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}",404) else diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2b0c672c7fa..f8c48e2f3b2 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -6,23 +6,23 @@ module API SUDO_PARAM = :sudo def current_user - @current_user ||= User.find_by_authentication_token(params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]) + private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + @current_user ||= User.find_by(authentication_token: private_token) identifier = sudo_identifier() + # If the sudo is the current user do nothing if (identifier && !(@current_user.id == identifier || @current_user.username == identifier)) render_api_error!('403 Forbidden: Must be admin to use sudo', 403) unless @current_user.is_admin? - begin - @current_user = User.by_username_or_id(identifier) - rescue => ex - not_found!("No user id or username for: #{identifier}") - end - not_found!("No user id or username for: #{identifier}") if current_user.nil? + @current_user = User.by_username_or_id(identifier) + not_found!("No user id or username for: #{identifier}") if @current_user.nil? end + @current_user end def sudo_identifier() identifier ||= params[SUDO_PARAM] ||= env[SUDO_HEADER] + # Regex for integers if (!!(identifier =~ /^[0-9]+$/)) identifier.to_i @@ -31,13 +31,23 @@ module API end end + def set_current_user_for_thread + Thread.current[:current_user] = current_user + + begin + yield + ensure + Thread.current[:current_user] = nil + end + end + def user_project @project ||= find_project(params[:id]) @project || not_found! end def find_project(id) - project = Project.find_by_id(id) || Project.find_with_namespace(id) + project = Project.find_by(id: id) || Project.find_with_namespace(id) if project && can?(current_user, :read_project, project) project diff --git a/lib/api/issues.rb b/lib/api/issues.rb index a15203d1563..3d15c35b8cc 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -2,7 +2,6 @@ module API # Issues API class Issues < Grape::API before { authenticate! } - before { Thread.current[:current_user] = current_user } resource :issues do # Get currently authenticated user's issues @@ -49,15 +48,17 @@ module API # Example Request: # POST /projects/:id/issues post ":id/issues" do - required_attributes! [:title] - attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id] - attrs[:label_list] = params[:labels] if params[:labels].present? - @issue = user_project.issues.new attrs - @issue.author = current_user - if @issue.save - present @issue, with: Entities::Issue - else - not_found! + set_current_user_for_thread do + required_attributes! [:title] + attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id] + attrs[:label_list] = params[:labels] if params[:labels].present? + @issue = user_project.issues.new attrs + @issue.author = current_user + if @issue.save + present @issue, with: Entities::Issue + else + not_found! + end end end @@ -75,16 +76,18 @@ module API # Example Request: # PUT /projects/:id/issues/:issue_id put ":id/issues/:issue_id" do - @issue = user_project.issues.find(params[:issue_id]) - authorize! :modify_issue, @issue + set_current_user_for_thread do + @issue = user_project.issues.find(params[:issue_id]) + authorize! :modify_issue, @issue - attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id, :state_event] - attrs[:label_list] = params[:labels] if params[:labels].present? + attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id, :state_event] + attrs[:label_list] = params[:labels] if params[:labels].present? - if @issue.update_attributes attrs - present @issue, with: Entities::Issue - else - not_found! + if @issue.update_attributes attrs + present @issue, with: Entities::Issue + else + not_found! + end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index d690f1d07e7..0f62cac9a0c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -2,7 +2,6 @@ module API # MergeRequest API class MergeRequests < Grape::API before { authenticate! } - before { Thread.current[:current_user] = current_user } resource :projects do helpers do @@ -70,28 +69,29 @@ module API # POST /projects/:id/merge_requests # post ":id/merge_requests" do - authorize! :write_merge_request, user_project - required_attributes! [:source_branch, :target_branch, :title] - attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :target_project_id] - merge_request = user_project.merge_requests.new(attrs) - merge_request.author = current_user - merge_request.source_project = user_project - target_project_id = attrs[:target_project_id] - if not_fork?(target_project_id, user_project) - merge_request.target_project = user_project - else - if target_matches_fork(target_project_id,user_project) - merge_request.target_project = Project.find_by_id(attrs[:target_project_id]) + set_current_user_for_thread do + authorize! :write_merge_request, user_project + required_attributes! [:source_branch, :target_branch, :title] + attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :target_project_id] + merge_request = user_project.merge_requests.new(attrs) + merge_request.author = current_user + merge_request.source_project = user_project + target_project_id = attrs[:target_project_id] + if not_fork?(target_project_id, user_project) + merge_request.target_project = user_project else - render_api_error!('(Bad Request) Specified target project that is not the source project, or the source fork of the project.', 400) + if target_matches_fork(target_project_id,user_project) + merge_request.target_project = Project.find_by(id: attrs[:target_project_id]) + else + render_api_error!('(Bad Request) Specified target project that is not the source project, or the source fork of the project.', 400) + end end - end - if merge_request.save - merge_request.reload_code - present merge_request, with: Entities::MergeRequest - else - handle_merge_request_errors! merge_request.errors + if merge_request.save + present merge_request, with: Entities::MergeRequest + else + handle_merge_request_errors! merge_request.errors + end end end @@ -109,17 +109,19 @@ module API # PUT /projects/:id/merge_request/:merge_request_id # put ":id/merge_request/:merge_request_id" do - attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :state_event] - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + set_current_user_for_thread do + attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :state_event] + merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :modify_merge_request, merge_request + authorize! :modify_merge_request, merge_request - if merge_request.update_attributes attrs - merge_request.reload_code - merge_request.mark_as_unchecked - present merge_request, with: Entities::MergeRequest - else - handle_merge_request_errors! merge_request.errors + if merge_request.update_attributes attrs + merge_request.reload_code + merge_request.mark_as_unchecked + present merge_request, with: Entities::MergeRequest + else + handle_merge_request_errors! merge_request.errors + end end end @@ -133,16 +135,18 @@ module API # POST /projects/:id/merge_request/:merge_request_id/comments # post ":id/merge_request/:merge_request_id/comments" do - required_attributes! [:note] + set_current_user_for_thread do + required_attributes! [:note] - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - note = merge_request.notes.new(note: params[:note], project_id: user_project.id) - note.author = current_user + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + note = merge_request.notes.new(note: params[:note], project_id: user_project.id) + note.author = current_user - if note.save - present note, with: Entities::MRNote - else - not_found! + if note.save + present note, with: Entities::MRNote + else + not_found! + end end end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index aee12e7dc40..f7e63b23093 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -40,15 +40,17 @@ module API # Example Request: # POST /projects/:id/milestones post ":id/milestones" do - authorize! :admin_milestone, user_project - required_attributes! [:title] + set_current_user_for_thread do + authorize! :admin_milestone, user_project + required_attributes! [:title] - attrs = attributes_for_keys [:title, :description, :due_date] - @milestone = user_project.milestones.new attrs - if @milestone.save - present @milestone, with: Entities::Milestone - else - not_found! + attrs = attributes_for_keys [:title, :description, :due_date] + @milestone = user_project.milestones.new attrs + if @milestone.save + present @milestone, with: Entities::Milestone + else + not_found! + end end end @@ -64,14 +66,16 @@ module API # Example Request: # PUT /projects/:id/milestones/:milestone_id put ":id/milestones/:milestone_id" do - authorize! :admin_milestone, user_project + set_current_user_for_thread do + authorize! :admin_milestone, user_project - @milestone = user_project.milestones.find(params[:milestone_id]) - attrs = attributes_for_keys [:title, :description, :due_date, :state_event] - if @milestone.update_attributes attrs - present @milestone, with: Entities::Milestone - else - not_found! + @milestone = user_project.milestones.find(params[:milestone_id]) + attrs = attributes_for_keys [:title, :description, :due_date, :state_event] + if @milestone.update_attributes attrs + present @milestone, with: Entities::Milestone + else + not_found! + end end end end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb new file mode 100644 index 00000000000..f9f2ed90ccc --- /dev/null +++ b/lib/api/namespaces.rb @@ -0,0 +1,23 @@ +module API + # namespaces API + class Namespaces < Grape::API + before { + authenticate! + authenticated_as_admin! + } + + resource :namespaces do + # Get a namespaces list + # + # Example Request: + # GET /namespaces + get do + @namespaces = Namespace.all + @namespaces = @namespaces.search(params[:search]) if params[:search].present? + @namespaces = paginate @namespaces + + present @namespaces, with: Entities::Namespace + end + end + end +end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index cb2bc764476..f21907b1ffc 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -41,17 +41,19 @@ module API # Example Request: # POST /projects/:id/notes post ":id/notes" do - required_attributes! [:body] + set_current_user_for_thread do + required_attributes! [:body] - @note = user_project.notes.new(note: params[:body]) - @note.author = current_user + @note = user_project.notes.new(note: params[:body]) + @note.author = current_user - if @note.save - present @note, with: Entities::Note - else - # :note is exposed as :body, but :note is set on error - bad_request!(:note) if @note.errors[:note].any? - not_found! + if @note.save + present @note, with: Entities::Note + else + # :note is exposed as :body, but :note is set on error + bad_request!(:note) if @note.errors[:note].any? + not_found! + end end end @@ -97,17 +99,19 @@ module API # POST /projects/:id/issues/:noteable_id/notes # POST /projects/:id/snippets/:noteable_id/notes post ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - required_attributes! [:body] + set_current_user_for_thread do + required_attributes! [:body] - @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) - @note = @noteable.notes.new(note: params[:body]) - @note.author = current_user - @note.project = user_project + @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) + @note = @noteable.notes.new(note: params[:body]) + @note.author = current_user + @note.project = user_project - if @note.save - present @note, with: Entities::Note - else - not_found! + if @note.save + present @note, with: Entities::Note + else + not_found! + end end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 738974955f3..c271dd8b61b 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -22,7 +22,7 @@ module API # GET /projects/:id/hooks get ":id/hooks" do @hooks = paginate user_project.hooks - present @hooks, with: Entities::Hook + present @hooks, with: Entities::ProjectHook end # Get a project hook @@ -34,7 +34,7 @@ module API # GET /projects/:id/hooks/:hook_id get ":id/hooks/:hook_id" do @hook = user_project.hooks.find(params[:hook_id]) - present @hook, with: Entities::Hook + present @hook, with: Entities::ProjectHook end @@ -47,10 +47,11 @@ module API # POST /projects/:id/hooks post ":id/hooks" do required_attributes! [:url] + attrs = attributes_for_keys [:url, :push_events, :issues_events, :merge_requests_events] + @hook = user_project.hooks.new(attrs) - @hook = user_project.hooks.new({"url" => params[:url]}) if @hook.save - present @hook, with: Entities::Hook + present @hook, with: Entities::ProjectHook else if @hook.errors[:url].present? error!("Invalid url given", 422) @@ -70,10 +71,10 @@ module API put ":id/hooks/:hook_id" do @hook = user_project.hooks.find(params[:hook_id]) required_attributes! [:url] + attrs = attributes_for_keys [:url, :push_events, :issues_events, :merge_requests_events] - attrs = attributes_for_keys [:url] if @hook.update_attributes attrs - present @hook, with: Entities::Hook + present @hook, with: Entities::ProjectHook else if @hook.errors[:url].present? error!("Invalid url given", 422) diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index bee6544ea3d..8e09fff6843 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -41,7 +41,6 @@ module API # id (required) - The ID of a project # title (required) - The title of a snippet # file_name (required) - The name of a snippet file - # lifetime (optional) - The expiration date of a snippet # code (required) - The content of a snippet # Example Request: # POST /projects/:id/snippets @@ -50,7 +49,6 @@ module API required_attributes! [:title, :file_name, :code] attrs = attributes_for_keys [:title, :file_name] - attrs[:expires_at] = params[:lifetime] if params[:lifetime].present? attrs[:content] = params[:code] if params[:code].present? @snippet = user_project.snippets.new attrs @snippet.author = current_user @@ -69,7 +67,6 @@ module API # snippet_id (required) - The ID of a project snippet # title (optional) - The title of a snippet # file_name (optional) - The name of a snippet file - # lifetime (optional) - The expiration date of a snippet # code (optional) - The content of a snippet # Example Request: # PUT /projects/:id/snippets/:snippet_id @@ -78,7 +75,6 @@ module API authorize! :modify_project_snippet, @snippet attrs = attributes_for_keys [:title, :file_name] - attrs[:expires_at] = params[:lifetime] if params[:lifetime].present? attrs[:content] = params[:code] if params[:code].present? if @snippet.update_attributes attrs diff --git a/lib/api/projects.rb b/lib/api/projects.rb index cf357b23c40..888aa7e77d2 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,6 +11,13 @@ module API end not_found! end + + def map_public_to_visibility_level(attrs) + publik = attrs.delete(:public) + publik = [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(publik) + attrs[:visibility_level] = Gitlab::VisibilityLevel::PUBLIC if !attrs[:visibility_level].present? && publik == true + attrs + end end # Get a projects list for authenticated user @@ -31,6 +38,16 @@ module API present @projects, with: Entities::Project end + # Get all projects for admin user + # + # Example Request: + # GET /projects/all + get '/all' do + authenticated_as_admin! + @projects = paginate Project + present @projects, with: Entities::Project + end + # Get a single project # # Parameters: @@ -60,14 +77,14 @@ module API # Parameters: # name (required) - name for new project # description (optional) - short project description - # default_branch (optional) - 'master' by default # issues_enabled (optional) # wall_enabled (optional) # merge_requests_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) # namespace_id (optional) - defaults to user namespace - # public (optional) - false by default + # public (optional) - if true same as setting visibility_level = 20 + # visibility_level (optional) - 0 by default # Example Request # POST /projects post do @@ -75,15 +92,17 @@ module API attrs = attributes_for_keys [:name, :path, :description, - :default_branch, :issues_enabled, :wall_enabled, :merge_requests_enabled, :wiki_enabled, :snippets_enabled, :namespace_id, - :public] - @project = ::Projects::CreateContext.new(current_user, attrs).execute + :public, + :visibility_level, + :import_url] + attrs = map_public_to_visibility_level(attrs) + @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? present @project, with: Entities::Project else @@ -106,7 +125,8 @@ module API # merge_requests_enabled (optional) # wiki_enabled (optional) # snippets_enabled (optional) - # public (optional) + # public (optional) - if true same as setting visibility_level = 20 + # visibility_level (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -120,8 +140,10 @@ module API :merge_requests_enabled, :wiki_enabled, :snippets_enabled, - :public] - @project = ::Projects::CreateContext.new(user, attrs).execute + :public, + :visibility_level] + attrs = map_public_to_visibility_level(attrs) + @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? present @project, with: Entities::Project else @@ -129,6 +151,16 @@ module API end end + # Remove project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # DELETE /projects/:id + delete ":id" do + authorize! :remove_project, user_project + user_project.destroy + end # Mark this project as forked from another # @@ -234,7 +266,7 @@ module API authorize! :admin_project, user_project required_attributes! [:access_level] - team_member = user_project.users_projects.find_by_user_id(params[:user_id]) + team_member = user_project.users_projects.find_by(user_id: params[:user_id]) not_found!("User can not be found") if team_member.nil? if team_member.update_attributes(project_access: params[:access_level]) @@ -254,7 +286,7 @@ module API # DELETE /projects/:id/members/:user_id delete ":id/members/:user_id" do authorize! :admin_project, user_project - team_member = user_project.users_projects.find_by_user_id(params[:user_id]) + team_member = user_project.users_projects.find_by(user_id: params[:user_id]) unless team_member.nil? team_member.destroy else @@ -272,7 +304,8 @@ module API # GET /projects/search/:query get "/search/:query" do ids = current_user.authorized_projects.map(&:id) - projects = Project.where("(id in (?) OR public = true) AND (name LIKE (?))", ids, "%#{params[:query]}%") + visibility_levels = [ Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PUBLIC ] + projects = Project.where("(id in (?) OR visibility_level in (?)) AND (name LIKE (?))", ids, visibility_levels, "%#{params[:query]}%") present paginate(projects), with: Entities::Project end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 1a911eae2bb..cad64760abb 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -1,3 +1,5 @@ +require 'mime/types' + module API # Projects API class Repositories < Grape::API @@ -49,7 +51,7 @@ module API @branch = user_project.repository.find_branch(params[:branch]) not_found! unless @branch - protected_branch = user_project.protected_branches.find_by_name(@branch.name) + protected_branch = user_project.protected_branches.find_by(name: @branch.name) user_project.protected_branches.create(name: @branch.name) unless protected_branch present @branch, with: Entities::RepoObject, project: user_project @@ -67,7 +69,7 @@ module API @branch = user_project.repository.find_branch(params[:branch]) not_found! unless @branch - protected_branch = user_project.protected_branches.find_by_name(@branch.name) + protected_branch = user_project.protected_branches.find_by(name: @branch.name) protected_branch.destroy if protected_branch present @branch, with: Entities::RepoObject, project: user_project @@ -80,7 +82,7 @@ module API # Example Request: # GET /projects/:id/repository/tags get ":id/repository/tags" do - present user_project.repo.tags.sort_by(&:name).reverse, with: Entities::RepoObject + present user_project.repo.tags.sort_by(&:name).reverse, with: Entities::RepoObject, project: user_project end # Get a project repository commits @@ -110,7 +112,7 @@ module API sha = params[:sha] commit = user_project.repository.commit(sha) not_found! "Commit" unless commit - present commit, with: Entities::RepoCommit + present commit, with: Entities::RepoCommitDetail end # Get the diff for a specific commit of a project @@ -122,9 +124,9 @@ module API # GET /projects/:id/repository/commits/:sha/diff get ":id/repository/commits/:sha/diff" do sha = params[:sha] - result = CommitLoadContext.new(user_project, current_user, {id: sha}).execute - not_found! "Commit" unless result[:commit] - result[:commit].diffs + commit = user_project.repository.commit(sha) + not_found! "Commit" unless commit + commit.diffs end # Get a project repository tree @@ -139,15 +141,9 @@ module API path = params[:path] || nil commit = user_project.repository.commit(ref) - tree = Tree.new(user_project.repository, commit.id, path) - - trees = [] - - %w(trees blobs submodules).each do |type| - trees += tree.send(type).map { |t| { name: t.name, type: type.singularize, mode: t.mode, id: t.id } } - end + tree = user_project.repository.tree(commit.id, path) - trees + present tree.sorted_entries, with: Entities::RepoTreeObject end # Get a raw file contents @@ -176,7 +172,58 @@ module API content_type blob.mime_type present blob.data end + + # Get a raw blob contents by blob sha + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The blob's sha + # Example Request: + # GET /projects/:id/repository/raw_blobs/:sha + get ":id/repository/raw_blobs/:sha" do + ref = params[:sha] + + repo = user_project.repository + + blob = Gitlab::Git::Blob.raw(repo, ref) + + not_found! "Blob" unless blob + + env['api.format'] = :txt + + content_type blob.mime_type + present blob.data + end + + # Get a an archive of the repository + # + # Parameters: + # id (required) - The ID of a project + # sha (optional) - the commit sha to download defaults to the tip of the default branch + # Example Request: + # GET /projects/:id/repository/archive + get ":id/repository/archive", requirements: { format: Gitlab::Regex.archive_formats_regex } do + authorize! :download_code, user_project + repo = user_project.repository + ref = params[:sha] + format = params[:format] + storage_path = Rails.root.join("tmp", "repositories") + + file_path = repo.archive_repo(ref, storage_path, format) + if file_path && File.exists?(file_path) + data = File.open(file_path, 'rb').read + + header["Content-Disposition"] = "attachment; filename=\"#{File.basename(file_path)}\"" + + content_type MIME::Types.type_for(file_path).first.content_type + + env['api.format'] = :binary + + present data + else + not_found! + end + end end end end - diff --git a/lib/api/services.rb b/lib/api/services.rb new file mode 100644 index 00000000000..bde502e32e1 --- /dev/null +++ b/lib/api/services.rb @@ -0,0 +1,44 @@ +module API + # Projects API + class Services < Grape::API + before { authenticate! } + before { authorize_admin_project } + + resource :projects do + # Set GitLab CI service for project + # + # Parameters: + # token (required) - CI project token + # project_url (required) - CI project url + # + # Example Request: + # PUT /projects/:id/services/gitlab-ci + put ":id/services/gitlab-ci" do + required_attributes! [:token, :project_url] + attrs = attributes_for_keys [:token, :project_url] + user_project.build_missing_services + + if user_project.gitlab_ci_service.update_attributes(attrs.merge(active: true)) + true + else + not_found! + end + end + + # Delete GitLab CI service settings + # + # Example Request: + # DELETE /projects/:id/keys/:id + delete ":id/services/gitlab-ci" do + if user_project.gitlab_ci_service + user_project.gitlab_ci_service.update_attributes( + active: false, + token: nil, + project_url: nil + ) + end + end + end + end +end + diff --git a/lib/api/users.rb b/lib/api/users.rb index 54d3aeecb70..ae808b6272b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -9,7 +9,7 @@ module API # Example Request: # GET /users get do - @users = User.scoped + @users = User.all @users = @users.active if params[:active].present? @users = @users.search(params[:search]) if params[:search].present? @users = paginate @users @@ -36,6 +36,7 @@ module API # skype - Skype ID # linkedin - Linkedin # twitter - Twitter account + # website_url - Website url # projects_limit - Number of projects user can create # extern_uid - External authentication provider UID # provider - External provider @@ -67,6 +68,7 @@ module API # skype - Skype ID # linkedin - Linkedin # twitter - Twitter account + # website_url - Website url # projects_limit - Limit projects each user can create # extern_uid - External authentication provider UID # provider - External provider @@ -78,7 +80,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :extern_uid, :provider, :bio, :can_create_group, :admin] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :extern_uid, :provider, :bio, :can_create_group, :admin] user = User.find(params[:id]) not_found!("User not found") unless user @@ -117,7 +119,7 @@ module API # DELETE /users/:id delete ":id" do authenticated_as_admin! - user = User.find_by_id(params[:id]) + user = User.find_by(id: params[:id]) if user user.destroy diff --git a/lib/backup/database.rb b/lib/backup/database.rb index c4fb2e2e159..ebb4f289c52 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -11,23 +11,29 @@ module Backup end def dump - case config["adapter"] + success = case config["adapter"] when /^mysql/ then - system("mysqldump #{mysql_args} #{config['database']} > #{db_file_name}") + print "Dumping MySQL database #{config['database']} ... " + system('mysqldump', *mysql_args, config['database'], out: db_file_name) when "postgresql" then + print "Dumping PostgreSQL database #{config['database']} ... " pg_env - system("pg_dump #{config['database']} > #{db_file_name}") + system('pg_dump', config['database'], out: db_file_name) end + report_success(success) end def restore - case config["adapter"] + success = case config["adapter"] when /^mysql/ then - system("mysql #{mysql_args} #{config['database']} < #{db_file_name}") + print "Restoring MySQL database #{config['database']} ... " + system('mysql', *mysql_args, config['database'], in: db_file_name) when "postgresql" then + print "Restoring PostgreSQL database #{config['database']} ... " pg_env - system("psql #{config['database']} -f #{db_file_name}") + system('psql', config['database'], '-f', db_file_name) end + report_success(success) end protected @@ -45,7 +51,7 @@ module Backup 'encoding' => '--default-character-set', 'password' => '--password' } - args.map { |opt, arg| "#{arg}='#{config[opt]}'" if config[opt] }.compact.join(' ') + args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact end def pg_env @@ -54,5 +60,13 @@ module Backup ENV['PGPORT'] = config["port"].to_s if config["port"] ENV['PGPASSWORD'] = config["password"].to_s if config["password"] end + + def report_success(success) + if success + puts '[DONE]'.green + else + puts '[FAILED]'.red + end + end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 258a0fb2589..efaefa4ce44 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,5 +1,7 @@ module Backup class Manager + BACKUP_CONTENTS = %w{repositories/ db/ uploads/ backup_information.yml} + def pack # saving additional informations s = {} @@ -16,7 +18,7 @@ module Backup # create archive print "Creating backup archive: #{s[:backup_created_at].to_i}_gitlab_backup.tar ... " - if Kernel.system("tar -cf #{s[:backup_created_at].to_i}_gitlab_backup.tar repositories/ db/ uploads/ backup_information.yml") + if Kernel.system('tar', '-cf', "#{s[:backup_created_at].to_i}_gitlab_backup.tar", *BACKUP_CONTENTS) puts "done".green else puts "failed".red @@ -25,7 +27,7 @@ module Backup def cleanup print "Deleting tmp directories ... " - if Kernel.system("rm -rf repositories/ db/ uploads/ backup_information.yml") + if Kernel.system('rm', '-rf', *BACKUP_CONTENTS) puts "done".green else puts "failed".red @@ -44,7 +46,7 @@ module Backup file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ } file_list.sort.each do |timestamp| if Time.at(timestamp) < (Time.now - keep_time) - if system("rm #{timestamp}_gitlab_backup.tar") + if Kernel.system(*%W(rm #{timestamp}_gitlab_backup.tar)) removed += 1 end end @@ -75,7 +77,7 @@ module Backup end print "Unpacking backup ... " - unless Kernel.system("tar -xf #{tar_file}") + unless Kernel.system(*%W(tar -xf #{tar_file})) puts "failed".red exit 1 else diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index c5e3d049fd7..20fd5ba92a1 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -18,7 +18,7 @@ module Backup # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace - if system("cd #{path_to_repo(project)} > /dev/null 2>&1 && git bundle create #{path_to_bundle(project)} --all > /dev/null 2>&1") + if system(*%W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all), silent) puts "[DONE]".green else puts "[FAILED]".red @@ -28,7 +28,9 @@ module Backup if File.exists?(path_to_repo(wiki)) print " * #{wiki.path_with_namespace} ... " - if system("cd #{path_to_repo(wiki)} > /dev/null 2>&1 && git bundle create #{path_to_bundle(wiki)} --all > /dev/null 2>&1") + if wiki.empty? + puts " [SKIPPED]".cyan + elsif system(*%W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all), silent) puts " [DONE]".green else puts " [FAILED]".red @@ -51,7 +53,7 @@ module Backup project.namespace.ensure_dir_exist if project.namespace - if system("git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)} > /dev/null 2>&1") + if system(*%W(git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)}), silent) puts "[DONE]".green else puts "[FAILED]".red @@ -61,7 +63,7 @@ module Backup if File.exists?(path_to_bundle(wiki)) print " * #{wiki.path_with_namespace} ... " - if system("git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)} > /dev/null 2>&1") + if system(*%W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}), silent) puts " [DONE]".green else puts " [FAILED]".red @@ -71,7 +73,7 @@ module Backup print 'Put GitLab hooks in repositories dirs'.yellow gitlab_shell_user_home = File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") - if system("#{gitlab_shell_user_home}/gitlab-shell/support/rewrite-hooks.sh #{Gitlab.config.gitlab_shell.repos_path}") + if system("#{gitlab_shell_user_home}/gitlab-shell/support/rewrite-hooks.sh", Gitlab.config.gitlab_shell.repos_path) puts " [DONE]".green else puts " [FAILED]".red @@ -101,5 +103,9 @@ module Backup FileUtils.rm_rf(backup_repos_path) FileUtils.mkdir_p(backup_repos_path) end + + def silent + {err: '/dev/null', out: '/dev/null'} + end end end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 462d3f1e274..e79da7e8fd2 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -19,7 +19,7 @@ module Backup FileUtils.cp_r(backup_uploads_dir, app_uploads_dir) end - + def backup_existing_uploads_dir if File.exists?(app_uploads_dir) FileUtils.mv(app_uploads_dir, Rails.root.join('public', "uploads.#{Time.now.to_i}")) diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 6e7872dcd03..e51cb30bdd9 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -117,7 +117,7 @@ module ExtractsPath end def tree - @tree ||= Tree.new(@repo, @commit.id, @path) + @tree ||= @repo.tree(@commit.id, @path) end private diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 0f196297477..955abc1bedd 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,7 +1,7 @@ module Gitlab class Auth def find(login, password) - user = User.find_by_email(login) || User.find_by_username(login) + user = User.find_by(email: login) || User.find_by(username: login) if user.nil? || user.ldap_user? # Second chance - try LDAP authentication diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index c522e0a413b..60c03ce1c04 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -38,6 +38,16 @@ module Grack # Authentication with username and password login, password = @auth.credentials + # Allow authentication for GitLab CI service + # if valid token passed + if login == "gitlab-ci-token" && project.gitlab_ci? + token = project.gitlab_ci_service.token + + if token.present? && token == password && service_name == 'git-upload-pack' + return @app.call(env) + end + end + @user = authenticate_user(login, password) if @user @@ -48,7 +58,7 @@ module Grack end else - return unauthorized unless project.public + return unauthorized unless project.public? end if authorized_git_request? @@ -59,14 +69,7 @@ module Grack end def authorized_git_request? - # Git upload and receive - if @request.get? - authorize_request(@request.params['service']) - elsif @request.post? - authorize_request(File.basename(@request.path)) - else - false - end + authorize_request(service_name) end def authenticate_user(login, password) @@ -77,29 +80,46 @@ module Grack def authorize_request(service) case service when 'git-upload-pack' - project.public || can?(user, :download_code, project) + can?(user, :download_code, project) when'git-receive-pack' - action = if project.protected_branch?(ref) - :push_code_to_protected_branches - else - :push_code - end + refs.each do |ref| + action = if project.protected_branch?(ref) + :push_code_to_protected_branches + else + :push_code + end + + return false unless can?(user, action, project) + end - can?(user, action, project) + # Never let git-receive-pack trough unauthenticated; it's + # harmless but git < 1.8 doesn't like it + return false if user.nil? + true else false end end + def service_name + if @request.get? + @request.params['service'] + elsif @request.post? + File.basename(@request.path) + else + nil + end + end + def project @project ||= project_by_path(@request.path_info) end - def ref - @ref ||= parse_ref + def refs + @refs ||= parse_refs end - def parse_ref + def parse_refs input = if @env["HTTP_CONTENT_ENCODING"] =~ /gzip/ Zlib::GzipReader.new(@request.body).read else @@ -108,7 +128,15 @@ module Grack # Need to reset seek point @request.body.rewind - /refs\/heads\/([\/\w\.-]+)/n.match(input.force_encoding('ascii-8bit')).to_a.last + + # Parse refs + refs = input.force_encoding('ascii-8bit').scan(/refs\/heads\/([\/\w\.-]+)/n).flatten.compact + + # Cleanup grabare from refs + # if push to multiple branches + refs.map do |ref| + ref.gsub(/00.*/, "") + end end end end diff --git a/lib/gitlab/backend/grack_helpers.rb b/lib/gitlab/backend/grack_helpers.rb index 5ac9e9f325b..cb747fe0137 100644 --- a/lib/gitlab/backend/grack_helpers.rb +++ b/lib/gitlab/backend/grack_helpers.rb @@ -1,7 +1,7 @@ module Grack module Helpers def project_by_path(path) - if m = /^\/([\w\.\/-]+)\.git/.match(path).to_a + if m = /^([\w\.\/-]+)\.git/.match(path).to_a path_with_namespace = m.last path_with_namespace.gsub!(/\.wiki$/, '') diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index c819ce56ac9..7121c8e40d2 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -10,7 +10,7 @@ module Gitlab # add_repository("gitlab/gitlab-ci") # def add_repository(name) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "add-project", "#{name}.git" + system "#{gitlab_shell_path}/bin/gitlab-projects", "add-project", "#{name}.git" end # Import repository @@ -21,7 +21,7 @@ module Gitlab # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(name, url) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "import-project", "#{name}.git", url + system "#{gitlab_shell_path}/bin/gitlab-projects", "import-project", "#{name}.git", url end # Move repository @@ -33,7 +33,7 @@ module Gitlab # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new.git") # def mv_repository(path, new_path) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "mv-project", "#{path}.git", "#{new_path}.git" + system "#{gitlab_shell_path}/bin/gitlab-projects", "mv-project", "#{path}.git", "#{new_path}.git" end # Update HEAD for repository @@ -45,7 +45,7 @@ module Gitlab # update_repository_head("gitlab/gitlab-ci", "3-1-stable") # def update_repository_head(path, branch) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "update-head", "#{path}.git", branch + system "#{gitlab_shell_path}/bin/gitlab-projects", "update-head", "#{path}.git", branch end # Fork repository to new namespace @@ -57,7 +57,7 @@ module Gitlab # fork_repository("gitlab/gitlab-ci", "randx") # def fork_repository(path, fork_namespace) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "fork-project", "#{path}.git", fork_namespace + system "#{gitlab_shell_path}/bin/gitlab-projects", "fork-project", "#{path}.git", fork_namespace end # Remove repository from file system @@ -68,7 +68,7 @@ module Gitlab # remove_repository("gitlab/gitlab-ci") # def remove_repository(name) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "rm-project", "#{name}.git" + system "#{gitlab_shell_path}/bin/gitlab-projects", "rm-project", "#{name}.git" end # Add repository branch from passed ref @@ -81,7 +81,7 @@ module Gitlab # add_branch("gitlab/gitlab-ci", "4-0-stable", "master") # def add_branch(path, branch_name, ref) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "create-branch", "#{path}.git", branch_name, ref + system "#{gitlab_shell_path}/bin/gitlab-projects", "create-branch", "#{path}.git", branch_name, ref end # Remove repository branch @@ -93,7 +93,7 @@ module Gitlab # rm_branch("gitlab/gitlab-ci", "4-0-stable") # def rm_branch(path, branch_name) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "rm-branch", "#{path}.git", branch_name + system "#{gitlab_shell_path}/bin/gitlab-projects", "rm-branch", "#{path}.git", branch_name end # Add repository tag from passed ref @@ -106,7 +106,7 @@ module Gitlab # add_tag("gitlab/gitlab-ci", "v4.0", "master") # def add_tag(path, tag_name, ref) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "create-tag", "#{path}.git", tag_name, ref + system "#{gitlab_shell_path}/bin/gitlab-projects", "create-tag", "#{path}.git", tag_name, ref end # Remove repository tag @@ -118,7 +118,7 @@ module Gitlab # rm_tag("gitlab/gitlab-ci", "v4.0") # def rm_tag(path, tag_name) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-projects", "rm-tag", "#{path}.git", tag_name + system "#{gitlab_shell_path}/bin/gitlab-projects", "rm-tag", "#{path}.git", tag_name end # Add new key to gitlab-shell @@ -127,7 +127,7 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-keys", "add-key", key_id, key_content + system "#{gitlab_shell_path}/bin/gitlab-keys", "add-key", key_id, key_content end # Remove ssh key from gitlab shell @@ -136,7 +136,7 @@ module Gitlab # remove_key("key-342", "sha-rsa ...") # def remove_key(key_id, key_content) - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-keys", "rm-key", key_id, key_content + system "#{gitlab_shell_path}/bin/gitlab-keys", "rm-key", key_id, key_content end # Remove all ssh keys from gitlab shell @@ -145,7 +145,7 @@ module Gitlab # remove_all_keys # def remove_all_keys - system "#{gitlab_shell_user_home}/gitlab-shell/bin/gitlab-keys", "clear" + system "#{gitlab_shell_path}/bin/gitlab-keys", "clear" end # Add empty directory for storing repositories @@ -196,8 +196,21 @@ module Gitlab Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" end + # Return GitLab shell version + def version + gitlab_shell_version_file = "#{gitlab_shell_path}/VERSION" + + if File.readable?(gitlab_shell_version_file) + File.read(gitlab_shell_version_file) + end + end + protected + def gitlab_shell_path + Gitlab.config.gitlab_shell.path + end + def gitlab_shell_user_home File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end diff --git a/lib/gitlab/blacklist.rb b/lib/gitlab/blacklist.rb index 2f9091e07df..6bc2c3b487c 100644 --- a/lib/gitlab/blacklist.rb +++ b/lib/gitlab/blacklist.rb @@ -3,7 +3,7 @@ module Gitlab extend self def path - %w(admin dashboard groups help profile projects search public assets u s teams merge_requests issues users snippets services repository hooks notes) + %w(admin dashboard files groups help profile projects search public assets u s teams merge_requests issues users snippets services repository hooks notes) end end end diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index a1ff248a77f..6e4de197eeb 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -6,17 +6,17 @@ module Gitlab if identifier.blank? # Local push from gitlab email = project.repository.commit(newrev).author_email rescue nil - User.find_by_email(email) if email + User.find_by(email: email) if email elsif identifier =~ /\Auser-\d+\Z/ # git push over http user_id = identifier.gsub("user-", "") - User.find_by_id(user_id) + User.find_by(id: user_id) elsif identifier =~ /\Akey-\d+\Z/ # git push over ssh key_id = identifier.gsub("key-", "") - Key.find_by_id(key_id).try(:user) + Key.find_by(id: key_id).try(:user) end end end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 78fc5dab9cb..fd36dda7d22 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -23,8 +23,8 @@ module Gitlab # Look for user with same emails # # Possible cases: - # * When user already has account and need to link his LDAP account. - # * LDAP uid changed for user with same email and we need to update his uid + # * When user already has account and need to link their LDAP account. + # * LDAP uid changed for user with same email and we need to update their uid # user = find_user(email) @@ -44,13 +44,13 @@ module Gitlab end def find_user(email) - user = model.find_by_email(email) + user = model.find_by(email: email) # If no user found and allow_username_or_email_login is true - # we look for user by extracting part of his email + # we look for user by extracting part of their email if !user && email && ldap_conf['allow_username_or_email_login'] uname = email.partition('@').first - user = model.find_by_username(uname) + user = model.find_by(username: uname) end user @@ -78,7 +78,7 @@ module Gitlab # * when ldap account was deactivated by change of OU membership in 'dn' def blocked?(dn) ldap = OmniAuth::LDAP::Adaptor.new(ldap_conf) - ldap.connection.search(base: dn, size: 1).blank? + ldap.connection.search(base: dn, scope: Net::LDAP::SearchScope_BaseObject, size: 1).blank? end private diff --git a/lib/gitlab/oauth/user.rb b/lib/gitlab/oauth/user.rb index 1b32b99f4ba..529753c4019 100644 --- a/lib/gitlab/oauth/user.rb +++ b/lib/gitlab/oauth/user.rb @@ -28,6 +28,7 @@ module Gitlab } user = model.build_user(opts, as: :admin) + user.skip_confirmation! user.save! log.info "(OAuth) Creating user #{email} from login with extern_uid => #{uid}" diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index b4be46d3b42..d18fc8bf2ce 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -18,10 +18,38 @@ module Gitlab default_regex end + def archive_formats_regex + #|zip|tar| tar.gz | tar.bz2 | + /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/ + end + + def git_reference_regex + # Valid git ref regex, see: + # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html + + %r{ + (?! + (?# doesn't begins with) + \/| (?# rule #6) + (?# doesn't contain) + .*(?: + [\/.]\.| (?# rule #1,3) + \/\/| (?# rule #6) + @\{| (?# rule #8) + \\ (?# rule #9) + ) + ) + [^\000-\040\177~^:?*\[]+ (?# rule #4-5) + (?# doesn't end with) + (?<!\.lock) (?# rule #1) + (?<![\/.]) (?# rule #6-7) + }x + end + protected def default_regex - /\A[a-zA-Z0-9][a-zA-Z0-9_\-\.]*\z/ + /\A[.?]?[a-zA-Z0-9][a-zA-Z0-9_\-\.]*(?<!\.git)\z/ end end end diff --git a/lib/gitlab/satellite/files/delete_file_action.rb b/lib/gitlab/satellite/files/delete_file_action.rb new file mode 100644 index 00000000000..30462999aa3 --- /dev/null +++ b/lib/gitlab/satellite/files/delete_file_action.rb @@ -0,0 +1,50 @@ +require_relative 'file_action' + +module Gitlab + module Satellite + class DeleteFileAction < FileAction + # Deletes file and creates a new commit for it + # + # Returns false if committing the change fails + # Returns false if pushing from the satellite to bare repo failed or was rejected + # Returns true otherwise + def commit!(content, commit_message) + in_locked_and_timed_satellite do |repo| + prepare_satellite!(repo) + + # create target branch in satellite at the corresponding commit from bare repo + repo.git.checkout({raise: true, timeout: true, b: true}, ref, "origin/#{ref}") + + # update the file in the satellite's working dir + file_path_in_satellite = File.join(repo.working_dir, file_path) + + # Prevent relative links + unless safe_path?(file_path_in_satellite) + Gitlab::GitLogger.error("FileAction: Relative path not allowed") + return false + end + + File.delete(file_path_in_satellite) + + # add removed file + repo.remove(file_path_in_satellite) + + # commit the changes + # will raise CommandFailed when commit fails + repo.git.commit(raise: true, timeout: true, a: true, m: commit_message) + + + # push commit back to bare repo + # will raise CommandFailed when push fails + repo.git.push({raise: true, timeout: true}, :origin, ref) + + # everything worked + true + end + rescue Grit::Git::CommandFailed => ex + Gitlab::GitLogger.error(ex.message) + false + end + end + end +end diff --git a/lib/gitlab/satellite/edit_file_action.rb b/lib/gitlab/satellite/files/edit_file_action.rb index d793d0ba8dc..cbdf70f7d12 100644 --- a/lib/gitlab/satellite/edit_file_action.rb +++ b/lib/gitlab/satellite/files/edit_file_action.rb @@ -1,40 +1,40 @@ +require_relative 'file_action' + module Gitlab module Satellite # GitLab server-side file update and commit - class EditFileAction < Action - attr_accessor :file_path, :ref - - def initialize(user, project, ref, file_path) - super user, project, git_timeout: 10.seconds - @file_path = file_path - @ref = ref - end - + class EditFileAction < FileAction # Updates the files content and creates a new commit for it # # Returns false if the ref has been updated while editing the file # Returns false if committing the change fails - # Returns false if pushing from the satellite to Gitolite failed or was rejected + # Returns false if pushing from the satellite to bare repo failed or was rejected # Returns true otherwise - def commit!(content, commit_message, last_commit) - return false unless can_edit?(last_commit) - + def commit!(content, commit_message, encoding) in_locked_and_timed_satellite do |repo| prepare_satellite!(repo) - # create target branch in satellite at the corresponding commit from Gitolite + # create target branch in satellite at the corresponding commit from bare repo repo.git.checkout({raise: true, timeout: true, b: true}, ref, "origin/#{ref}") # update the file in the satellite's working dir file_path_in_satellite = File.join(repo.working_dir, file_path) - File.open(file_path_in_satellite, 'w') { |f| f.write(content) } + + # Prevent relative links + unless safe_path?(file_path_in_satellite) + Gitlab::GitLogger.error("FileAction: Relative path not allowed") + return false + end + + # Write file + write_file(file_path_in_satellite, content, encoding) # commit the changes # will raise CommandFailed when commit fails repo.git.commit(raise: true, timeout: true, a: true, m: commit_message) - # push commit back to Gitolite + # push commit back to bare repo # will raise CommandFailed when push fails repo.git.push({raise: true, timeout: true}, :origin, ref) @@ -45,13 +45,6 @@ module Gitlab Gitlab::GitLogger.error(ex.message) false end - - protected - - def can_edit?(last_commit) - current_last_commit = Gitlab::Git::Commit.last_for_path(@project.repository, ref, file_path).sha - last_commit == current_last_commit - end end end end diff --git a/lib/gitlab/satellite/files/file_action.rb b/lib/gitlab/satellite/files/file_action.rb new file mode 100644 index 00000000000..7701a6d5d60 --- /dev/null +++ b/lib/gitlab/satellite/files/file_action.rb @@ -0,0 +1,25 @@ +module Gitlab + module Satellite + class FileAction < Action + attr_accessor :file_path, :ref + + def initialize(user, project, ref, file_path) + super user, project, git_timeout: 10.seconds + @file_path = file_path + @ref = ref + end + + def safe_path?(path) + File.absolute_path(path) == path + end + + def write_file(abs_file_path, content, file_encoding = 'text') + if file_encoding == 'base64' + File.open(abs_file_path, 'wb') { |f| f.write(Base64.decode64(content)) } + else + File.open(abs_file_path, 'w') { |f| f.write(content) } + end + end + end + end +end diff --git a/lib/gitlab/satellite/files/new_file_action.rb b/lib/gitlab/satellite/files/new_file_action.rb new file mode 100644 index 00000000000..15e9b7a6f77 --- /dev/null +++ b/lib/gitlab/satellite/files/new_file_action.rb @@ -0,0 +1,55 @@ +require_relative 'file_action' + +module Gitlab + module Satellite + class NewFileAction < FileAction + # Updates the files content and creates a new commit for it + # + # Returns false if the ref has been updated while editing the file + # Returns false if committing the change fails + # Returns false if pushing from the satellite to bare repo failed or was rejected + # Returns true otherwise + def commit!(content, commit_message, encoding) + in_locked_and_timed_satellite do |repo| + prepare_satellite!(repo) + + # create target branch in satellite at the corresponding commit from bare repo + repo.git.checkout({raise: true, timeout: true, b: true}, ref, "origin/#{ref}") + + file_path_in_satellite = File.join(repo.working_dir, file_path) + dir_name_in_satellite = File.dirname(file_path_in_satellite) + + # Prevent relative links + unless safe_path?(file_path_in_satellite) + Gitlab::GitLogger.error("FileAction: Relative path not allowed") + return false + end + + # Create dir if not exists + FileUtils.mkdir_p(dir_name_in_satellite) + + # Write file + write_file(file_path_in_satellite, content, encoding) + + # add new file + repo.add(file_path_in_satellite) + + # commit the changes + # will raise CommandFailed when commit fails + repo.git.commit(raise: true, timeout: true, a: true, m: commit_message) + + + # push commit back to bare repo + # will raise CommandFailed when push fails + repo.git.push({raise: true, timeout: true}, :origin, ref) + + # everything worked + true + end + rescue Grit::Git::CommandFailed => ex + Gitlab::GitLogger.error(ex.message) + false + end + end + end +end diff --git a/lib/gitlab/satellite/merge_action.rb b/lib/gitlab/satellite/merge_action.rb index d74d4194ff6..85615f282c4 100644 --- a/lib/gitlab/satellite/merge_action.rb +++ b/lib/gitlab/satellite/merge_action.rb @@ -24,11 +24,11 @@ module Gitlab # Returns false if the merge produced conflicts # Returns false if pushing from the satellite to the repository failed or was rejected # Returns true otherwise - def merge! + def merge!(merge_commit_message = nil) in_locked_and_timed_satellite do |merge_repo| prepare_satellite!(merge_repo) - if merge_in_satellite!(merge_repo) - # push merge back to Gitolite + if merge_in_satellite!(merge_repo, merge_commit_message) + # push merge back to bare repo # will raise CommandFailed when push fails merge_repo.git.push(default_options, :origin, merge_request.target_branch) # remove source branch @@ -49,14 +49,7 @@ module Gitlab in_locked_and_timed_satellite do |merge_repo| prepare_satellite!(merge_repo) update_satellite_source_and_target!(merge_repo) - - if merge_request.for_fork? - diff = merge_repo.git.native(:diff, default_options, "origin/#{merge_request.target_branch}", "source/#{merge_request.source_branch}") - else - diff = merge_repo.git.native(:diff, default_options, "#{merge_request.target_branch}", "#{merge_request.source_branch}") - end - - return diff + diff = merge_repo.git.native(:diff, default_options, "origin/#{merge_request.target_branch}", "source/#{merge_request.source_branch}") end rescue Grit::Git::CommandFailed => ex handle_exception(ex) @@ -88,14 +81,7 @@ module Gitlab in_locked_and_timed_satellite do |merge_repo| prepare_satellite!(merge_repo) update_satellite_source_and_target!(merge_repo) - - if (merge_request.for_fork?) - patch = merge_repo.git.format_patch(default_options({stdout: true}), "origin/#{merge_request.target_branch}..source/#{merge_request.source_branch}") - else - patch = merge_repo.git.format_patch(default_options({stdout: true}), "#{merge_request.target_branch}..#{merge_request.source_branch}") - end - - return patch + patch = merge_repo.git.format_patch(default_options({stdout: true}), "origin/#{merge_request.target_branch}..source/#{merge_request.source_branch}") end rescue Grit::Git::CommandFailed => ex handle_exception(ex) @@ -125,34 +111,26 @@ module Gitlab # # Returns false if the merge produced conflicts # Returns true otherwise - def merge_in_satellite!(repo) + def merge_in_satellite!(repo, message = nil) update_satellite_source_and_target!(repo) + message ||= "Merge branch '#{merge_request.source_branch}' into '#{merge_request.target_branch}'" + # merge the source branch into the satellite # will raise CommandFailed when merge fails - if merge_request.for_fork? - repo.git.pull(default_options({no_ff: true}), 'source', merge_request.source_branch) - else - repo.git.pull(default_options({no_ff: true}), 'origin', merge_request.source_branch) - end + repo.git.merge(default_options({no_ff: true}), "-m #{message}", "source/#{merge_request.source_branch}") rescue Grit::Git::CommandFailed => ex handle_exception(ex) end # Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for merges, diffs etc def update_satellite_source_and_target!(repo) - if merge_request.for_fork? - repo.remote_add('source', merge_request.source_project.repository.path_to_repo) - repo.remote_fetch('source') - repo.git.checkout(default_options({b: true}), merge_request.target_branch, "origin/#{merge_request.target_branch}") - else - repo.git.checkout(default_options, "#{merge_request.source_branch}") - repo.git.checkout(default_options({t: true}), "origin/#{merge_request.target_branch}") - end + repo.remote_add('source', merge_request.source_project.repository.path_to_repo) + repo.remote_fetch('source') + repo.git.checkout(default_options({b: true}), merge_request.target_branch, "origin/#{merge_request.target_branch}") rescue Grit::Git::CommandFailed => ex handle_exception(ex) end - end end end diff --git a/lib/gitlab/satellite/satellite.rb b/lib/gitlab/satellite/satellite.rb index 6cb7814fae5..353c3024aad 100644 --- a/lib/gitlab/satellite/satellite.rb +++ b/lib/gitlab/satellite/satellite.rb @@ -123,7 +123,7 @@ module Gitlab remotes.each { |name| repo.git.remote(default_options,'rm', name)} end - # Updates the satellite from Gitolite + # Updates the satellite from bare repo # # Note: this will only update remote branches (i.e. origin/*) def update_from_source! diff --git a/lib/gitlab/theme.rb b/lib/gitlab/theme.rb index 89604162304..44237a062fc 100644 --- a/lib/gitlab/theme.rb +++ b/lib/gitlab/theme.rb @@ -15,7 +15,7 @@ module Gitlab COLOR => "ui_color" } - id ||= 1 + id ||= Gitlab.config.gitlab.default_theme return themes[id] end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb new file mode 100644 index 00000000000..f46685e4bbe --- /dev/null +++ b/lib/gitlab/upgrader.rb @@ -0,0 +1,96 @@ +require_relative "version_info" + +module Gitlab + class Upgrader + def execute + puts "GitLab #{current_version.major} upgrade tool" + puts "Your version is #{current_version}" + puts "Latest available version for GitLab #{current_version.major} is #{latest_version}" + + if latest_version? + puts "You are using the latest GitLab version" + else + puts "Newer GitLab version is available" + answer = if ARGV.first == "-y" + "yes" + else + prompt("Do you want to upgrade (yes/no)? ", %w{yes no}) + end + + if answer == "yes" + upgrade + else + exit 0 + end + end + end + + def latest_version? + current_version >= latest_version + end + + def current_version + @current_version ||= Gitlab::VersionInfo.parse(current_version_raw) + end + + def latest_version + @latest_version ||= Gitlab::VersionInfo.parse(latest_version_raw) + end + + def current_version_raw + File.read(File.join(gitlab_path, "VERSION")).strip + end + + def latest_version_raw + git_tags = `git ls-remote --tags origin | grep tags\/v#{current_version.major}` + git_tags = git_tags.lines.to_a.select { |version| version =~ /v\d\.\d\.\d\Z/ } + last_tag = git_tags.last.match(/v\d\.\d\.\d/).to_s + end + + def update_commands + { + "Stash changed files" => "git stash", + "Get latest code" => "git fetch", + "Switch to new version" => "git checkout v#{latest_version}", + "Install gems" => "bundle", + "Migrate DB" => "bundle exec rake db:migrate RAILS_ENV=production", + "Recompile assets" => "bundle exec rake assets:clean assets:precompile RAILS_ENV=production", + "Clear cache" => "bundle exec rake cache:clear RAILS_ENV=production" + } + end + + def upgrade + update_commands.each do |title, cmd| + puts title + puts " -> #{cmd}" + if system(cmd) + puts " -> OK" + else + puts " -> FAILED" + puts "Failed to upgrade. Try to repeat task or proceed with upgrade manually " + exit 1 + end + end + + puts "Done" + end + + def gitlab_path + File.expand_path(File.join(File.dirname(__FILE__), '../..')) + end + + # Prompt the user to input something + # + # message - the message to display before input + # choices - array of strings of acceptable answers or nil for any answer + # + # Returns the user's answer + def prompt(message, choices = nil) + begin + print(message) + answer = STDIN.gets.chomp + end while !choices.include?(answer) + answer + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb new file mode 100644 index 00000000000..eada9bcddf5 --- /dev/null +++ b/lib/gitlab/visibility_level.rb @@ -0,0 +1,42 @@ +# Gitlab::VisibilityLevel module +# +# Define allowed public modes that can be used for +# GitLab projects to determine project public mode +# +module Gitlab + module VisibilityLevel + PRIVATE = 0 + INTERNAL = 10 + PUBLIC = 20 + + class << self + def values + options.values + end + + def options + { + 'Private' => PRIVATE, + 'Internal' => INTERNAL, + 'Public' => PUBLIC + } + end + + def allowed_for?(user, level) + user.is_admin? || !Gitlab.config.gitlab.restricted_visibility_levels.include?(level) + end + end + + def private? + visibility_level_field == PRIVATE + end + + def internal? + visibility_level_field == INTERNAL + end + + def public? + visibility_level_field == PUBLIC + end + end +end diff --git a/lib/redcarpet/render/gitlab_html.rb b/lib/redcarpet/render/gitlab_html.rb index d9c2d3b626d..6da0c1d6f96 100644 --- a/lib/redcarpet/render/gitlab_html.rb +++ b/lib/redcarpet/render/gitlab_html.rb @@ -6,14 +6,12 @@ class Redcarpet::Render::GitlabHTML < Redcarpet::Render::HTML def initialize(template, options = {}) @template = template @project = @template.instance_variable_get("@project") + @ref = @template.instance_variable_get("@ref") + @request_path = @template.instance_variable_get("@path") super options end def block_code(code, language) - options = { options: {encoding: 'utf-8'} } - lexer = Pygments::Lexer.find(language) # language can be an alias - options.merge!(lexer: lexer.aliases[0].downcase) if lexer # downcase is required - # New lines are placed to fix an rendering issue # with code wrapped inside <h1> tag for next case: # @@ -23,7 +21,11 @@ class Redcarpet::Render::GitlabHTML < Redcarpet::Render::HTML # <<-HTML - <div class="#{h.user_color_scheme_class}">#{Pygments.highlight(code, options)}</div> +<div class="highlighted-data #{h.user_color_scheme_class}"> + <div class="highlight"> + <pre><code class="#{language}">#{code}</code></pre> + </div> +</div> HTML end @@ -32,7 +34,19 @@ class Redcarpet::Render::GitlabHTML < Redcarpet::Render::HTML h.link_to_gfm(content, link, title: title) end + def preprocess(full_document) + if @project + h.create_relative_links(full_document, @project, @ref, @request_path, is_wiki?) + else + full_document + end + end + def postprocess(full_document) h.gfm(full_document) end + + def is_wiki? + @template.instance_variable_get("@wiki") + end end diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh index 0d2f8418bcf..b96f73058b6 100755 --- a/lib/support/deploy/deploy.sh +++ b/lib/support/deploy/deploy.sh @@ -28,17 +28,18 @@ sudo -u git -H git pull origin master echo 'Deploy: Bundle and migrate' # change it to your needs -sudo -u git -H bundle --without postgres +sudo -u git -H bundle --without aws development test postgres --deployment sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production +sudo -u git -H bundle exec rake assets:clean RAILS_ENV=production +sudo -u git -H bundle exec rake assets:precompile RAILS_ENV=production +sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production # return stashed changes (if necessary) # sudo -u git -H git stash pop - echo 'Deploy: Starting GitLab server...' sudo service gitlab start -sleep 10 sudo -u git -H rm /home/git/gitlab/public/index.html -echo 'Deploy: Done' +echo 'Deploy: Done'
\ No newline at end of file diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 6aff7b5a8f9..c6e570784e0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -3,7 +3,6 @@ # GITLAB # Maintainer: @randx # Authors: rovanion.luckey@gmail.com, @randx -# App Version: 6.0 ### BEGIN INIT INFO # Provides: gitlab @@ -15,36 +14,44 @@ # Description: GitLab git repository management ### END INIT INFO + +### +# DO NOT EDIT THIS FILE! +# This file will be overwritten on update. +# Instead add/change your variables in /etc/default/gitlab +# An example defaults file can be found in lib/support/init.d/gitlab.default.example +### + + ### Environment variables RAILS_ENV="production" -# Script variable names should be lower-case not to conflict with internal -# /bin/sh variables such as PATH, EDITOR or SHELL. -app_root="/home/git/gitlab" +# Script variable names should be lower-case not to conflict with +# internal /bin/sh variables such as PATH, EDITOR or SHELL. app_user="git" -unicorn_conf="$app_root/config/unicorn.rb" +app_root="/home/$app_user/gitlab" pid_path="$app_root/tmp/pids" socket_path="$app_root/tmp/sockets" web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" - - -### Here ends user configuration ### - +# Read configuration variable file if it is present +test -f /etc/default/gitlab && . /etc/default/gitlab # Switch to the app_user if it is not he/she who is running the script. if [ "$USER" != "$app_user" ]; then sudo -u "$app_user" -H -i $0 "$@"; exit; fi -# Switch to the gitlab path, if it fails exit with an error. +# Switch to the gitlab path, exit on failure. if ! cd "$app_root" ; then echo "Failed to cd into $app_root, exiting!"; exit 1 fi + ### Init Script functions +## Gets the pids from the files check_pids(){ if ! mkdir -p "$pid_path"; then echo "Could not create the path $pid_path needed to store the pids." @@ -63,12 +70,29 @@ check_pids(){ fi } +## Called when we have started the two processes and are waiting for their pid files. +wait_for_pids(){ + # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid + i=0; + while [ ! -f $web_server_pid_path -o ! -f $sidekiq_pid_path ]; do + sleep 0.1; + i=$((i+1)) + if [ $((i%10)) = 0 ]; then + echo -n "." + elif [ $((i)) = 301 ]; then + echo "Waited 30s for the processes to write their pids, something probably went wrong." + exit 1; + fi + done + echo +} + # We use the pids in so many parts of the script it makes sense to always check them. # Only after start() is run should the pids change. Sidekiq sets it's own pid. check_pids -# Checks whether the different parts of the service are already running or not. +## Checks whether the different parts of the service are already running or not. check_status(){ check_pids # If the web server is running kill -0 $wpid returns true, or rather 0. @@ -85,9 +109,16 @@ check_status(){ else sidekiq_status="-1" fi + if [ $web_status = 0 -a $sidekiq_status = 0 ]; then + gitlab_status=0 + else + # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html + # code 3 means 'program is not running' + gitlab_status=3 + fi } -# Check for stale pids and remove them if necessary +## Check for stale pids and remove them if necessary. check_stale_pids(){ check_status # If there is a pid it is something else than 0, the service is running if @@ -95,7 +126,7 @@ check_stale_pids(){ if [ "$wpid" != "0" -a "$web_status" != "0" ]; then echo "Removing stale Unicorn web server pid. This is most likely caused by the web server crashing the last time it ran." if ! rm "$web_server_pid_path"; then - echo "Unable to remove stale pid, exiting" + echo "Unable to remove stale pid, exiting." exit 1 fi fi @@ -108,7 +139,7 @@ check_stale_pids(){ fi } -# If no parts of the service is running, bail out. +## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then @@ -117,86 +148,92 @@ exit_if_not_running(){ fi } -# Starts Unicorn and Sidekiq. +## Starts Unicorn and Sidekiq if they're not running. start() { check_stale_pids + if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then + echo -n "Starting both the GitLab Unicorn and Sidekiq" + elif [ "$web_status" != "0" ]; then + echo -n "Starting GitLab Sidekiq" + elif [ "$sidekiq_status" != "0" ]; then + echo -n "Starting GitLab Unicorn" + fi + # Then check if the service is running. If it is: don't start again. if [ "$web_status" = "0" ]; then echo "The Unicorn web server already running with pid $wpid, not restarting." else - echo "Starting the GitLab Unicorn web server..." # Remove old socket if it exists rm -f "$socket_path"/gitlab.socket 2>/dev/null - # Start the webserver - bundle exec unicorn_rails -D -c "$unicorn_conf" -E "$RAILS_ENV" + # Start the web server + RAILS_ENV=$RAILS_ENV script/web start & fi # If sidekiq is already running, don't start it again. if [ "$sidekiq_status" = "0" ]; then echo "The Sidekiq job dispatcher is already running with pid $spid, not restarting" else - echo "Starting the GitLab Sidekiq event dispatcher..." - RAILS_ENV=$RAILS_ENV bundle exec rake sidekiq:start - # We are sleeping a bit here because sidekiq is slow at writing it's pid - sleep 2 + RAILS_ENV=$RAILS_ENV script/background_jobs start & fi + # Wait for the pids to be planted + wait_for_pids # Finally check the status to tell wether or not GitLab is running - status + print_status } -# Asks the Unicorn and the Sidekiq if they would be so kind as to stop, if not kills them. +## Asks the Unicorn and the Sidekiq if they would be so kind as to stop, if not kills them. stop() { exit_if_not_running + + if [ "$web_status" = "0" -a "$sidekiq_status" = "0" ]; then + echo -n "Shutting down both Unicorn and Sidekiq" + elif [ "$web_status" = "0" ]; then + echo -n "Shutting down Sidekiq" + elif [ "$sidekiq_status" = "0" ]; then + echo -n "Shutting down Unicorn" + fi + # If the Unicorn web server is running, tell it to stop; if [ "$web_status" = "0" ]; then - kill -QUIT "$wpid" - echo "Stopping the GitLab Unicorn web server..." - stopping=true - else - echo "The Unicorn web was not running, doing nothing." + RAILS_ENV=$RAILS_ENV script/web stop fi # And do the same thing for the Sidekiq. if [ "$sidekiq_status" = "0" ]; then - printf "Stopping Sidekiq job dispatcher." - RAILS_ENV=$RAILS_ENV bundle exec rake sidekiq:stop - stopping=true - else - echo "The Sidekiq was not running, must have run out of breath." + RAILS_ENV=$RAILS_ENV script/background_jobs stop fi - # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$stopping" = "true" ]; do + while [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; do sleep 1 check_status - if [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; then - printf "." - else + printf "." + if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then printf "\n" break fi done + sleep 1 # Cleaning up unused pids rm "$web_server_pid_path" 2>/dev/null # rm "$sidekiq_pid_path" # Sidekiq seems to be cleaning up it's own pid. - status + print_status } -# Returns the status of GitLab and it's components -status() { +## Prints the status of GitLab and it's components. +print_status() { check_status if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then echo "GitLab is not running." return fi if [ "$web_status" = "0" ]; then - echo "The GitLab Unicorn webserver with pid $wpid is running." + echo "The GitLab Unicorn web server with pid $wpid is running." else - printf "The GitLab Unicorn webserver is \033[31mnot running\033[0m.\n" + printf "The GitLab Unicorn web server is \033[31mnot running\033[0m.\n" fi if [ "$sidekiq_status" = "0" ]; then echo "The GitLab Sidekiq job dispatcher with pid $spid is running." @@ -208,6 +245,7 @@ status() { fi } +## Tells unicorn to reload it's config and Sidekiq to restart reload(){ exit_if_not_running if [ "$wpid" = "0" ];then @@ -215,17 +253,16 @@ reload(){ exit 1 fi printf "Reloading GitLab Unicorn configuration... " - kill -USR2 "$wpid" + RAILS_ENV=$RAILS_ENV script/web reload echo "Done." echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." - RAILS_ENV=$RAILS_ENV bundle exec rake sidekiq:stop - echo "Starting Sidekiq..." - RAILS_ENV=$RAILS_ENV bundle exec rake sidekiq:start - # Waiting 2 seconds for sidekiq to write it. - sleep 2 - status + RAILS_ENV=$RAILS_ENV script/background_jobs restart + + wait_for_pids + print_status } +## Restarts Sidekiq and Unicorn. restart(){ check_status if [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; then @@ -235,7 +272,7 @@ restart(){ } -## Finally the input handling. +### Finally the input handling. case "$1" in start) @@ -251,7 +288,8 @@ case "$1" in reload ;; status) - status + print_status + exit $gitlab_status ;; *) echo "Usage: service gitlab {start|stop|restart|reload|status}" diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example new file mode 100755 index 00000000000..9951bacedf5 --- /dev/null +++ b/lib/support/init.d/gitlab.default.example @@ -0,0 +1,31 @@ +# Copy this lib/support/init.d/gitlab.default.example file to +# /etc/default/gitlab in order for it to apply to your system. + +# RAILS_ENV defines the type of installation that is running. +# Normal values are "production", "test" and "development". +RAILS_ENV="production" + +# app_user defines the user that GitLab is run as. +# The default is "git". +app_user="git" + +# app_root defines the folder in which gitlab and it's components are installed. +# The default is "/home/$app_user/gitlab" +app_root="/home/$app_user/gitlab" + +# pid_path defines a folder in which the gitlab and it's components place their pids. +# This variable is also used below to define the relevant pids for the gitlab components. +# The default is "$app_root/tmp/pids" +pid_path="$app_root/tmp/pids" + +# socket_path defines the folder in which gitlab places the sockets +#The default is "$app_root/tmp/sockets" +socket_path="$app_root/tmp/sockets" + +# web_server_pid_path defines the path in which to create the pid file fo the web_server +# The default is "$pid_path/unicorn.pid" +web_server_pid_path="$pid_path/unicorn.pid" + +# sidekiq_pid_path defines the path in which to create the pid file for sidekiq +# The default is "$pid_path/sidekiq.pid" +sidekiq_pid_path="$pid_path/sidekiq.pid" diff --git a/lib/support/logrotate/gitlab b/lib/support/logrotate/gitlab new file mode 100644 index 00000000000..df9398d0795 --- /dev/null +++ b/lib/support/logrotate/gitlab @@ -0,0 +1,22 @@ +# GitLab logrotate settings +# based on: http://stackoverflow.com/a/4883967 + +/home/git/gitlab/log/*.log { + weekly + missingok + rotate 52 + compress + delaycompress + notifempty + copytruncate +} + +/home/git/gitlab-shell/gitlab-shell.log { + weekly + missingok + rotate 52 + compress + delaycompress + notifempty + copytruncate +} diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 3e929c52990..882f0386046 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -1,6 +1,19 @@ # GITLAB # Maintainer: @randx -# App Version: 5.0 + +# CHUNKED TRANSFER +# It is a known issue that Git-over-HTTP requires chunked transfer encoding [0] which is not +# supported by Nginx < 1.3.9 [1]. As a result, pushing a large object with Git (i.e. a single large file) +# can lead to a 411 error. In theory you can get around this by tweaking this configuration file and either +# - installing an old version of Nginx with the chunkin module [2] compiled in, or +# - using a newer version of Nginx. +# +# At the time of writing we do not know if either of these theoretical solutions works. As a workaround +# users can use Git over SSH to push large files. +# +# [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99 +# [1] https://github.com/agentzh/chunkin-nginx-module#status +# [2] https://github.com/agentzh/chunkin-nginx-module upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket; @@ -11,6 +24,10 @@ server { server_name YOUR_SERVER_FQDN; # e.g., server_name source.example.com; server_tokens off; # don't show the version number, a security best practice root /home/git/gitlab/public; + + # Increase this if you want to upload large attachments + # Or if you want to accept large git objects over http + client_max_body_size 5m; # individual nginx logs for this gitlab vhost access_log /var/log/nginx/gitlab_access.log; @@ -25,15 +42,18 @@ server { # if a file, which is not found in the root folder is requested, # then the proxy pass the request to the upsteam (gitlab unicorn) location @gitlab { - proxy_read_timeout 300; # https://github.com/gitlabhq/gitlabhq/issues/694 - proxy_connect_timeout 300; # https://github.com/gitlabhq/gitlabhq/issues/694 + proxy_read_timeout 300; # Some requests take more than 30 seconds. + proxy_connect_timeout 300; # Some requests take more than 30 seconds. proxy_redirect off; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_pass http://gitlab; } + + error_page 502 /502.html; } diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index c270232edba..612a9ba93a8 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -15,7 +15,7 @@ namespace :gitlab do desc "GITLAB | Add a specific user to all projects (as a developer)" task :user_to_projects, [:email] => :environment do |t, args| - user = User.find_by_email args.email + user = User.find_by(email: args.email) project_ids = Project.pluck(:id) puts "Importing #{user.email} users into #{project_ids.size} projects" UsersProject.add_users_into_projects(project_ids, Array.wrap(user.id), UsersProject::DEVELOPER) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index f52b98cb3ae..c91dedf74c7 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -3,6 +3,7 @@ namespace :gitlab do task check: %w{gitlab:env:check gitlab:gitlab_shell:check gitlab:sidekiq:check + gitlab:ldap:check gitlab:app:check} @@ -278,8 +279,6 @@ namespace :gitlab do start_checking "Environment" check_gitlab_git_config - check_python2_exists - check_python2_version finished_checking "Environment" end @@ -313,52 +312,6 @@ namespace :gitlab do fix_and_rerun end end - - def check_python2_exists - print "Has python2? ... " - - # Python prints its version to STDERR - # so we can't just use run("python2 --version") - if run_and_match("which python2", /python2$/) - puts "yes".green - else - puts "no".red - try_fixing_it( - "Make sure you have Python 2.5+ installed", - "Link it to python2" - ) - for_more_information( - see_installation_guide_section "Packages / Dependencies" - ) - fix_and_rerun - end - end - - def check_python2_version - print "python2 is supported version? ... " - - # Python prints its version to STDERR - # so we can't just use run("python2 --version") - - unless run_and_match("which python2", /python2$/) - puts "can't check because of previous errors".magenta - return - end - - if `python2 --version 2>&1` =~ /2\.[567]\.\d/ - puts "yes".green - else - puts "no".red - try_fixing_it( - "Make sure you have Python 2.5+ installed", - "Link it to python2" - ) - for_more_information( - see_installation_guide_section "Packages / Dependencies" - ) - fix_and_rerun - end - end end @@ -611,10 +564,7 @@ namespace :gitlab do end def gitlab_shell_version - gitlab_shell_version_file = "#{gitlab_shell_user_home}/gitlab-shell/VERSION" - if File.readable?(gitlab_shell_version_file) - File.read(gitlab_shell_version_file) - end + Gitlab::Shell.new.version end def has_gitlab_shell3? @@ -643,12 +593,12 @@ namespace :gitlab do def check_sidekiq_running print "Running? ... " - if sidekiq_process_match + if sidekiq_process_count > 0 puts "yes".green else puts "no".red try_fixing_it( - sudo_gitlab("bundle exec rake sidekiq:start RAILS_ENV=production") + sudo_gitlab("RAILS_ENV=production script/background_jobs start") ) for_more_information( see_installation_guide_section("Install Init Script"), @@ -659,14 +609,14 @@ namespace :gitlab do end def only_one_sidekiq_running - sidekiq_match = sidekiq_process_match - return unless sidekiq_match + process_count = sidekiq_process_count + return if process_count.zero? print 'Number of Sidekiq processes ... ' - if sidekiq_match.length == 1 + if process_count == 1 puts '1'.green else - puts "#{sidekiq_match.length}".red + puts "#{process_count}".red try_fixing_it( 'sudo service gitlab stop', "sudo pkill -u #{gitlab_user} -f sidekiq", @@ -677,11 +627,51 @@ namespace :gitlab do end end - def sidekiq_process_match - run_and_match("ps ux | grep -i sidekiq", /(sidekiq \d+\.\d+\.\d+.+$)/) + def sidekiq_process_count + `ps ux`.scan(/sidekiq \d+\.\d+\.\d+/).count end end + namespace :ldap do + task :check, [:limit] => :environment do |t, args| + # Only show up to 100 results because LDAP directories can be very big. + # This setting only affects the `rake gitlab:check` script. + args.with_defaults(limit: 100) + warn_user_is_not_gitlab + start_checking "LDAP" + + if ldap_config.enabled + print_users(args.limit) + else + puts 'LDAP is disabled in config/gitlab.yml' + end + + finished_checking "LDAP" + end + + def print_users(limit) + puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" + ldap.search(attributes: attributes, filter: filter, size: limit, return_result: false) do |entry| + puts "DN: #{entry.dn}\t#{ldap_config.uid}: #{entry[ldap_config.uid]}" + end + end + + def attributes + [ldap_config.uid] + end + + def filter + Net::LDAP::Filter.present?(ldap_config.uid) + end + + def ldap + @ldap ||= OmniAuth::LDAP::Adaptor.new(ldap_config).connection + end + + def ldap_config + @ldap_config ||= Gitlab.config.ldap + end + end # Helper methods ########################## @@ -736,7 +726,7 @@ namespace :gitlab do end def check_gitlab_shell - required_version = Gitlab::VersionInfo.new(1, 7, 1) + required_version = Gitlab::VersionInfo.new(1, 7, 9) current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) print "GitLab Shell version >= #{required_version} ? ... " diff --git a/lib/tasks/gitlab/enable_namespaces.rake b/lib/tasks/gitlab/enable_namespaces.rake index 927748c0fd5..201f34ab546 100644 --- a/lib/tasks/gitlab/enable_namespaces.rake +++ b/lib/tasks/gitlab/enable_namespaces.rake @@ -43,13 +43,13 @@ namespace :gitlab do username.gsub!("+", ".") # return username if no matches - return username unless User.find_by_username(username) + return username unless User.find_by(username: username) # look for same username (1..10).each do |i| suffixed_username = "#{username}#{i}" - return suffixed_username unless User.find_by_username(suffixed_username) + return suffixed_username unless User.find_by(username: suffixed_username) end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 8fa89270854..cbfa736c84c 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,12 +2,12 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy your bare repos under git repos_path - # 2. run bundle exec rake gitlab:import:repos RAILS_ENV=production + # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: - # * project owner will be a first admin - # * existing projects will be skipped + # * The project owner will set to the first administator of the system + # * Existing projects will be skipped # desc "GITLAB | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" task repos: :environment do @@ -50,7 +50,7 @@ namespace :gitlab do # find group namespace if group_name - group = Group.find_by_path(group_name) + group = Group.find_by(path: group_name) # create group namespace if !group group = Group.new(:name => group_name) @@ -66,7 +66,7 @@ namespace :gitlab do project_params[:namespace_id] = group.id end - project = Projects::CreateContext.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute if project.valid? puts " * Created #{project.name} (#{repo_path})".green diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ac2c4577c77..c46d5855faf 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,6 +2,16 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end +unless STDOUT.isatty + module Colored + extend self + + def colorize(string, options={}) + string + end + end +end + namespace :gitlab do # Ask if the user wants to continue diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake index d0e9dfe46a1..e91678473a8 100644 --- a/lib/tasks/sidekiq.rake +++ b/lib/tasks/sidekiq.rake @@ -1,20 +1,21 @@ namespace :sidekiq do desc "GITLAB | Stop sidekiq" task :stop do - system "bundle exec sidekiqctl stop #{pidfile}" + system "script/background_jobs stop" end desc "GITLAB | Start sidekiq" task :start do - system "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,gitlab_shell,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &" + system "script/background_jobs start" end - desc "GITLAB | Start sidekiq with launchd on Mac OS X" - task :launchd do - system "bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,gitlab_shell,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1" + desc 'GitLab | Restart sidekiq' + task :restart do + system "script/background_jobs restart" end - def pidfile - Rails.root.join("tmp", "pids", "sidekiq.pid") + desc "GITLAB | Start sidekiq with launchd on Mac OS X" + task :launchd do + system "script/background_jobs start_no_deamonize" end end |