diff options
Diffstat (limited to 'lib')
28 files changed, 500 insertions, 164 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 4a67313430a..6c5391b98c8 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -50,6 +50,67 @@ module API not_found! "Commit" unless commit commit.diffs end + + # Get a commit's comments + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The commit hash + # Examples: + # GET /projects/:id/repository/commits/:sha/comments + get ':id/repository/commits/:sha/comments' do + sha = params[:sha] + commit = user_project.repository.commit(sha) + not_found! 'Commit' unless commit + notes = Note.where(commit_id: commit.id) + present paginate(notes), with: Entities::CommitNote + end + + # Post comment to commit + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The commit hash + # note (required) - Text of comment + # path (optional) - The file path + # line (optional) - The line number + # line_type (optional) - The type of line (new or old) + # Examples: + # POST /projects/:id/repository/commits/:sha/comments + post ':id/repository/commits/:sha/comments' do + required_attributes! [:note] + + sha = params[:sha] + commit = user_project.repository.commit(sha) + not_found! 'Commit' unless commit + opts = { + note: params[:note], + noteable_type: 'Commit', + commit_id: commit.id + } + + if params[:path] && params[:line] && params[:line_type] + commit.diffs.each do |diff| + next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.lines.to_a) + + lines.each do |line| + next unless line.new_pos == params[:line].to_i && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + end + + break if opts[:line_code] + end + end + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: Entities::CommitNote + else + not_found! + end + end end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 40696489ba2..2fea151aeb3 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -14,10 +14,14 @@ module API expose :bio, :skype, :linkedin, :twitter, :website_url end + class Identity < Grape::Entity + expose :provider, :extern_uid + end + class UserFull < User expose :email - expose :theme_id, :color_scheme_id, :extern_uid, :provider, \ - :projects_limit + expose :theme_id, :color_scheme_id, :projects_limit + expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project end @@ -179,6 +183,14 @@ module API expose :author, using: Entities::UserBasic end + class CommitNote < Grape::Entity + expose :note + expose(:path) { |note| note.diff_file_name } + expose(:line) { |note| note.diff_new_line } + expose(:line_type) { |note| note.diff_line_type } + expose :author, using: Entities::UserBasic + end + class Event < Grape::Entity expose :title, :project_id, :action_name expose :target_id, :target_type, :author_id diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ebf2296097d..180e50611cf 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -33,17 +33,22 @@ module API end project = Project.find_with_namespace(project_path) - return false unless project + + unless project + return Gitlab::GitAccessStatus.new(false, 'No such project') + end actor = if params[:key_id] - Key.find(params[:key_id]) + Key.find_by(id: params[:key_id]) elsif params[:user_id] - User.find(params[:user_id]) + User.find_by(id: params[:user_id]) end - return false unless actor + unless actor + return Gitlab::GitAccessStatus.new(false, 'No such user or key') + end - access.allowed?( + access.check( actor, params[:action], project, diff --git a/lib/api/users.rb b/lib/api/users.rb index d07815a8a97..37b36ddcf94 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -59,10 +59,16 @@ module API post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :extern_uid, :provider, :bio, :can_create_group, :admin] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :can_create_group, :admin] user = User.build_user(attrs) admin = attrs.delete(:admin) user.admin = admin unless admin.nil? + + identity_attrs = attributes_for_keys [:provider, :extern_uid] + if identity_attrs.any? + user.identities.build(identity_attrs) + end + if user.save present user, with: Entities::UserFull else @@ -89,8 +95,6 @@ module API # twitter - Twitter account # website_url - Website url # projects_limit - Limit projects each user can create - # extern_uid - External authentication provider UID - # provider - External provider # bio - Bio # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false @@ -99,7 +103,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :extern_uid, :provider, :bio, :can_create_group, :admin] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :can_create_group, :admin] user = User.find(params[:id]) not_found!('User') unless user diff --git a/lib/backup/database.rb b/lib/backup/database.rb index d12d30a9110..9ab6aca276d 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -13,10 +13,10 @@ module Backup def dump success = case config["adapter"] when /^mysql/ then - print "Dumping MySQL database #{config['database']} ... " + $progress.print "Dumping MySQL database #{config['database']} ... " system('mysqldump', *mysql_args, config['database'], out: db_file_name) when "postgresql" then - print "Dumping PostgreSQL database #{config['database']} ... " + $progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env system('pg_dump', config['database'], out: db_file_name) end @@ -27,13 +27,14 @@ module Backup def restore success = case config["adapter"] when /^mysql/ then - print "Restoring MySQL database #{config['database']} ... " + $progress.print "Restoring MySQL database #{config['database']} ... " system('mysql', *mysql_args, config['database'], in: db_file_name) when "postgresql" then - print "Restoring PostgreSQL database #{config['database']} ... " + $progress.print "Restoring PostgreSQL database #{config['database']} ... " # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE # statements like MySQL. Rake::Task["gitlab:db:drop_all_tables"].invoke + Rake::Task["gitlab:db:drop_all_postgres_sequences"].invoke pg_env system('psql', config['database'], '-f', db_file_name) end @@ -68,9 +69,9 @@ module Backup def report_success(success) if success - puts '[DONE]'.green + $progress.puts '[DONE]'.green else - puts '[FAILED]'.red + $progress.puts '[FAILED]'.red end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 03fe0f0b02f..ab8db4e9837 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -18,11 +18,11 @@ module Backup end # create archive - print "Creating backup archive: #{tar_file} ... " + $progress.print "Creating backup archive: #{tar_file} ... " if Kernel.system('tar', '-cf', tar_file, *BACKUP_CONTENTS) - puts "done".green + $progress.puts "done".green else - puts "failed".red + puts "creating archive #{tar_file} failed".red abort 'Backup failed' end @@ -31,37 +31,37 @@ module Backup def upload(tar_file) remote_directory = Gitlab.config.backup.upload.remote_directory - print "Uploading backup archive to remote storage #{remote_directory} ... " + $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection if connection_settings.blank? - puts "skipped".yellow + $progress.puts "skipped".yellow return end connection = ::Fog::Storage.new(connection_settings) directory = connection.directories.get(remote_directory) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false) - puts "done".green + $progress.puts "done".green else - puts "failed".red + puts "uploading backup to #{remote_directory} failed".red abort 'Backup failed' end end def cleanup - print "Deleting tmp directories ... " + $progress.print "Deleting tmp directories ... " if Kernel.system('rm', '-rf', *BACKUP_CONTENTS) - puts "done".green + $progress.puts "done".green else - puts "failed".red + puts "deleting tmp directory failed".red abort 'Backup failed' end end def remove_old # delete backups - print "Deleting old backups ... " + $progress.print "Deleting old backups ... " keep_time = Gitlab.config.backup.keep_time.to_i path = Gitlab.config.backup.path @@ -76,9 +76,9 @@ module Backup end end end - puts "done. (#{removed} removed)".green + $progress.puts "done. (#{removed} removed)".green else - puts "skipping".yellow + $progress.puts "skipping".yellow end end @@ -101,12 +101,12 @@ module Backup exit 1 end - print "Unpacking backup ... " + $progress.print "Unpacking backup ... " unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "failed".red + puts "unpacking backup failed".red exit 1 else - puts "done".green + $progress.puts "done".green end settings = YAML.load_file("backup_information.yml") diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 380beac708d..e18bc804437 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -8,19 +8,21 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - print " * #{project.path_with_namespace} ... " + $progress.print " * #{project.path_with_namespace} ... " # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace if project.empty_repo? - puts "[SKIPPED]".cyan + $progress.puts "[SKIPPED]".cyan else - output, status = Gitlab::Popen.popen(%W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all)) + cmd = %W(git --git-dir=#{path_to_repo(project)} bundle create #{path_to_bundle(project)} --all) + output, status = Gitlab::Popen.popen(cmd) if status.zero? - puts "[DONE]".green + $progress.puts "[DONE]".green else puts "[FAILED]".red + puts "failed: #{cmd.join(' ')}" puts output abort 'Backup failed' end @@ -29,15 +31,17 @@ module Backup wiki = ProjectWiki.new(project) if File.exists?(path_to_repo(wiki)) - print " * #{wiki.path_with_namespace} ... " + $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? - puts " [SKIPPED]".cyan + $progress.puts " [SKIPPED]".cyan else - output, status = Gitlab::Popen.popen(%W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)) + cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + output, status = Gitlab::Popen.popen(cmd) if status.zero? - puts " [DONE]".green + $progress.puts " [DONE]".green else puts " [FAILED]".red + puts "failed: #{cmd.join(' ')}" abort 'Backup failed' end end @@ -55,35 +59,52 @@ module Backup FileUtils.mkdir_p(repos_path) Project.find_each(batch_size: 1000) do |project| - print "#{project.path_with_namespace} ... " + $progress.print " * #{project.path_with_namespace} ... " project.namespace.ensure_dir_exist if project.namespace - if system(*%W(git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)}), silent) - puts "[DONE]".green + if File.exists?(path_to_bundle(project)) + cmd = %W(git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)}) + else + cmd = %W(git init --bare #{path_to_repo(project)}) + end + + if system(*cmd, silent) + $progress.puts "[DONE]".green else puts "[FAILED]".red + puts "failed: #{cmd.join(' ')}" abort 'Restore failed' end wiki = ProjectWiki.new(project) if File.exists?(path_to_bundle(wiki)) - print " * #{wiki.path_with_namespace} ... " - if system(*%W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}), silent) - puts " [DONE]".green + $progress.print " * #{wiki.path_with_namespace} ... " + + # If a wiki bundle exists, first remove the empty repo + # that was initialized with ProjectWiki.new() and then + # try to restore with 'git clone --bare'. + FileUtils.rm_rf(path_to_repo(wiki)) + cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + + if system(*cmd, silent) + $progress.puts " [DONE]".green else puts " [FAILED]".red + puts "failed: #{cmd.join(' ')}" abort 'Restore failed' end end end - print 'Put GitLab hooks in repositories dirs'.yellow - if system("#{Gitlab.config.gitlab_shell.path}/bin/create-hooks") - puts " [DONE]".green + $progress.print 'Put GitLab hooks in repositories dirs'.yellow + cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" + if system(cmd) + $progress.puts " [DONE]".green else puts " [FAILED]".red + puts "failed: #{cmd}" end end @@ -91,7 +112,7 @@ module Backup protected def path_to_repo(project) - File.join(repos_path, project.path_with_namespace + '.git') + project.repository.path_to_repo end def path_to_bundle(project) diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index df1461a45c9..762639414e0 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -80,7 +80,7 @@ module Grack case git_cmd when *Gitlab::GitAccess::DOWNLOAD_COMMANDS if user - Gitlab::GitAccess.new.download_allowed?(user, project) + Gitlab::GitAccess.new.download_access_check(user, project).allowed? elsif project.public? # Allow clone/fetch for public projects true diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index cc320da751c..aabc7f1e69a 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -8,6 +8,13 @@ module Gitlab end end + class << self + def version_required + @version_required ||= File.read(Rails.root. + join('GITLAB_SHELL_VERSION')).strip + end + end + # Init new repository # # name - project path with namespace diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb new file mode 100644 index 00000000000..6a52cdba608 --- /dev/null +++ b/lib/gitlab/force_push_check.rb @@ -0,0 +1,15 @@ +module Gitlab + class ForcePushCheck + def self.force_push?(project, oldrev, newrev) + return false if project.empty_repo? + + if oldrev != Gitlab::Git::BLANK_SHA && newrev != Gitlab::Git::BLANK_SHA + missed_refs = IO.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})).read + missed_refs.split("\n").size > 0 + else + false + end + end + end +end + diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 129881060d5..875f8d8b3a3 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -5,61 +5,77 @@ module Gitlab attr_reader :params, :project, :git_cmd, :user - def allowed?(actor, cmd, project, changes = nil) + def check(actor, cmd, project, changes = nil) case cmd when *DOWNLOAD_COMMANDS + download_access_check(actor, project) + when *PUSH_COMMANDS if actor.is_a? User - download_allowed?(actor, project) + push_access_check(actor, project, changes) elsif actor.is_a? DeployKey - actor.projects.include?(project) + return build_status_object(false, "Deploy key not allowed to push") elsif actor.is_a? Key - download_allowed?(actor.user, project) + push_access_check(actor.user, project, changes) else raise 'Wrong actor' end - when *PUSH_COMMANDS - if actor.is_a? User - push_allowed?(actor, project, changes) - elsif actor.is_a? DeployKey - # Deploy key not allowed to push - return false - elsif actor.is_a? Key - push_allowed?(actor.user, project, changes) + else + return build_status_object(false, "Wrong command") + end + end + + def download_access_check(actor, project) + if actor.is_a?(User) + user_download_access_check(actor, project) + elsif actor.is_a?(DeployKey) + if actor.projects.include?(project) + build_status_object(true) else - raise 'Wrong actor' + build_status_object(false, "Deploy key not allowed to access this project") end + elsif actor.is_a? Key + user_download_access_check(actor.user, project) else - false + raise 'Wrong actor' end end - def download_allowed?(user, project) - if user && user_allowed?(user) - user.can?(:download_code, project) + def user_download_access_check(user, project) + if user && user_allowed?(user) && user.can?(:download_code, project) + build_status_object(true) else - false + build_status_object(false, "You don't have access") end end - def push_allowed?(user, project, changes) - return false unless user && user_allowed?(user) - return true if changes.blank? + def push_access_check(user, project, changes) + unless user && user_allowed?(user) + return build_status_object(false, "You don't have access") + end + + if changes.blank? + return build_status_object(true) + end + + unless project.repository.exists? + return build_status_object(false, "Repository does not exist") + end changes = changes.lines if changes.kind_of?(String) # Iterate over all changes to find if user allowed all of them to be applied changes.each do |change| - unless change_allowed?(user, project, change) + status = change_access_check(user, project, change) + unless status.allowed? # If user does not have access to make at least one change - cancel all push - return false + return status end end - # If user has access to make all changes - true + return build_status_object(true) end - def change_allowed?(user, project, change) + def change_access_check(user, project, change) oldrev, newrev, ref = change.split(' ') action = if project.protected_branch?(branch_name(ref)) @@ -72,25 +88,22 @@ module Gitlab else :push_code_to_protected_branches end - elsif project.repository && project.repository.tag_names.include?(tag_name(ref)) + elsif project.repository.tag_names.include?(tag_name(ref)) # Prevent any changes to existing git tag unless user has permissions :admin_project else :push_code end - user.can?(action, project) + if user.can?(action, project) + build_status_object(true) + else + build_status_object(false, "You don't have permission") + end end def forced_push?(project, oldrev, newrev) - return false if project.empty_repo? - - if oldrev != Gitlab::Git::BLANK_SHA && newrev != Gitlab::Git::BLANK_SHA - missed_refs = IO.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev})).read - missed_refs.split("\n").size > 0 - else - false - end + Gitlab::ForcePushCheck.force_push?(project, oldrev, newrev) end private @@ -116,5 +129,11 @@ module Gitlab nil end end + + protected + + def build_status_object(status, message = '') + GitAccessStatus.new(status, message) + end end end diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb new file mode 100644 index 00000000000..3d451ecebee --- /dev/null +++ b/lib/gitlab/git_access_status.rb @@ -0,0 +1,15 @@ +module Gitlab + class GitAccessStatus + attr_accessor :status, :message + alias_method :allowed?, :status + + def initialize(status, message = '') + @status = status + @message = message + end + + def to_json + {status: @status, message: @message}.to_json + end + end +end
\ No newline at end of file diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 9f0eb3be20f..a2177c8d548 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,7 +1,11 @@ module Gitlab class GitAccessWiki < GitAccess - def change_allowed?(user, project, change) - user.can?(:write_wiki, project) + def change_access_check(user, project, change) + if user.can?(:write_wiki, project) + build_status_object(true) + else + build_status_object(false, "You don't have access") + end end end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 0fdd4dbe577..39d17def930 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -6,7 +6,7 @@ module Gitlab # Returns true for a valid reference name, false otherwise def validate(ref_name) Gitlab::Utils.system_silent( - %W(git check-ref-format refs/#{ref_name})) == 0 + %W(git check-ref-format refs/#{ref_name})) end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index eb2c4e48ff2..0c85acf7e69 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :adapter, :provider, :user def self.open(user, &block) - Gitlab::LDAP::Adapter.open(user.provider) do |adapter| + Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| block.call(self.new(user, adapter)) end end @@ -28,13 +28,13 @@ module Gitlab def initialize(user, adapter=nil) @adapter = adapter @user = user - @provider = user.provider + @provider = user.ldap_identity.provider end def allowed? - if Gitlab::LDAP::Person.find_by_dn(user.extern_uid, adapter) + if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) return true unless ldap_config.active_directory - !Gitlab::LDAP::Person.disabled_via_active_directory?(user.extern_uid, adapter) + !Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) else false end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 3176e9790a7..3ef494ba137 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -12,9 +12,10 @@ module Gitlab class << self def find_by_uid_and_provider(uid, provider) # LDAP distinguished name is case-insensitive - ::User. + identity = ::Identity. where(provider: [provider, :ldap]). where('lower(extern_uid) = ?', uid.downcase).last + identity && identity.user end end @@ -34,15 +35,13 @@ module Gitlab end def find_by_email - model.find_by(email: auth_hash.email) + ::User.find_by(email: auth_hash.email) end def update_user_attributes - gl_user.attributes = { - extern_uid: auth_hash.uid, - provider: auth_hash.provider, - email: auth_hash.email - } + gl_user.email = auth_hash.email + gl_user.identities.build(provider: auth_hash.provider, extern_uid: auth_hash.uid) + gl_user end def changed? diff --git a/lib/gitlab/oauth/user.rb b/lib/gitlab/oauth/user.rb index 47f62153a50..6861427864e 100644 --- a/lib/gitlab/oauth/user.rb +++ b/lib/gitlab/oauth/user.rb @@ -5,6 +5,8 @@ # module Gitlab module OAuth + class ForbiddenAction < StandardError; end + class User attr_accessor :auth_hash, :gl_user @@ -70,24 +72,24 @@ module Gitlab end def find_by_uid_and_provider - model.where(provider: auth_hash.provider, extern_uid: auth_hash.uid).last + identity = Identity.find_by(provider: auth_hash.provider, extern_uid: auth_hash.uid) + identity && identity.user end def build_new_user - model.new(user_attributes).tap do |user| - user.skip_confirmation! - end + user = ::User.new(user_attributes) + user.skip_confirmation! + user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) + user end def user_attributes { - extern_uid: auth_hash.uid, - provider: auth_hash.provider, name: auth_hash.name, username: auth_hash.username, email: auth_hash.email, password: auth_hash.password, - password_confirmation: auth_hash.password, + password_confirmation: auth_hash.password } end @@ -95,12 +97,8 @@ module Gitlab Gitlab::AppLogger end - def model - ::User - end - - def raise_unauthorized_to_create - raise StandardError.new("Unauthorized to create user, signup disabled for #{auth_hash.provider}") + def unauthorized_to_create + raise ForbiddenAction.new("Unauthorized to create user, signup disabled for #{auth_hash.provider}") end end end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb new file mode 100644 index 00000000000..0f2db50e98c --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -0,0 +1,53 @@ +module Gitlab + module SidekiqMiddleware + class MemoryKiller + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # Create a mutex used to ensure there will be only one thread waiting to + # shut Sidekiq down + MUTEX = Mutex.new + + def call(worker, job, queue) + yield + current_rss = get_rss + + return unless MAX_RSS > 0 && current_rss > MAX_RSS + + Thread.new do + # Return if another thread is already waiting to shut Sidekiq down + return unless MUTEX.try_lock + + Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ + "#{MAX_RSS}" + Sidekiq.logger.warn "spawned thread that will shut down PID "\ + "#{Process.pid} in #{GRACE_TIME} seconds" + sleep(GRACE_TIME) + + Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" + Process.kill('SIGUSR1', Process.pid) + + Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ + "SIGTERM to PID #{Process.pid}" + sleep(SHUTDOWN_WAIT) + + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" + Process.kill('SIGTERM', Process.pid) + end + end + + private + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{Process.pid})) + return 0 unless status.zero? + + output.to_i + end + end + end +end diff --git a/lib/gitlab/theme.rb b/lib/gitlab/theme.rb index b7c50cb734d..a7c83a880f6 100644 --- a/lib/gitlab/theme.rb +++ b/lib/gitlab/theme.rb @@ -19,5 +19,19 @@ module Gitlab return themes[id] end + + def self.type_css_class_by_id(id) + types = { + BASIC => 'light_theme', + MARS => 'dark_theme', + MODERN => 'dark_theme', + GRAY => 'dark_theme', + COLOR => 'dark_theme' + } + + id ||= Gitlab.config.gitlab.default_theme + + types[id] + end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index bc30364550a..bd184c27187 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -5,10 +5,9 @@ module Gitlab # Run system command without outputting to stdout. # # @param cmd [Array<String>] - # @return [Integer] exit status + # @return [Boolean] def system_silent(cmd) - IO.popen(cmd).close - $?.exitstatus + Popen::popen(cmd).last.zero? end end end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 49a68c62293..c8b769ace8e 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -1,5 +1,5 @@ ## GitLab -## Maintainer: @randx +## Contributors: randx, yin8086, sashkab, orkoden, axilleas, bbodenmiller ## ## Lines starting with two hashes (##) are comments with information. ## Lines starting with one hash (#) are configuration parameters that can be uncommented. @@ -15,7 +15,7 @@ ## - installing an old version of Nginx with the chunkin module [2] compiled in, or ## - using a newer version of Nginx. ## -## At the time of writing we do not know if either of these theoretical solutions works. +## At the time of writing we do not know if either of these theoretical solutions works. ## As a workaround users can use Git over SSH to push large files. ## ## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99 @@ -26,6 +26,7 @@ ## configuration ## ################################### ## +## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; @@ -33,7 +34,8 @@ upstream gitlab { ## Normal HTTP host server { - listen *:80 default_server; + listen 0.0.0.0:80 default_server; + listen [::]:80 default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice root /home/git/gitlab/public; @@ -42,6 +44,8 @@ server { ## Or if you want to accept large git objects over http client_max_body_size 20m; + ## See app/controllers/application_controller.rb for headers set + ## Individual nginx logs for this GitLab vhost access_log /var/log/nginx/gitlab_access.log; error_log /var/log/nginx/gitlab_error.log; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index cbb198086b5..4e53d5e8b50 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -1,5 +1,5 @@ ## GitLab -## Contributors: randx, yin8086, sashkab, orkoden, axilleas +## Contributors: randx, yin8086, sashkab, orkoden, axilleas, bbodenmiller ## ## Modified from nginx http version ## Modified from http://blog.phusion.nl/2012/04/21/tutorial-setting-up-gitlab-on-debian-6/ @@ -26,9 +26,8 @@ ## [1] https://github.com/agentzh/chunkin-nginx-module#status ## [2] https://github.com/agentzh/chunkin-nginx-module ## -## ################################### -## SSL configuration ## +## configuration ## ################################### ## ## See installation.md#using-https for additional HTTPS configuration details. @@ -37,22 +36,24 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -## Normal HTTP host +## Redirects all HTTP traffic to the HTTPS host server { - listen *:80 default_server; + listen 0.0.0.0:80; + listen [::]:80 default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice - - ## Redirects all traffic to the HTTPS host - root /nowhere; ## root doesn't have to be a valid path since we are redirecting - rewrite ^ https://$server_name$request_uri? permanent; + return 301 https://$server_name$request_uri; + access_log /var/log/nginx/gitlab_access.log; + error_log /var/log/nginx/gitlab_error.log; } + ## HTTPS host server { - listen 443 ssl; + listen 0.0.0.0:443 ssl; + listen [::]:443 ssl default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com - server_tokens off; + server_tokens off; ## Don't show the nginx version number, a security best practice root /home/git/gitlab/public; ## Increase this if you want to upload large attachments @@ -70,12 +71,9 @@ server { ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_prefer_server_ciphers on; ssl_session_cache shared:SSL:10m; + ssl_session_timeout 5m; - ## [WARNING] The following header states that the browser should only communicate - ## with your server over a secure connection for the next 24 months. - add_header Strict-Transport-Security max-age=63072000; - add_header X-Frame-Options SAMEORIGIN; - add_header X-Content-Type-Options nosniff; + ## See app/controllers/application_controller.rb for headers set ## [Optional] If your certficate has OCSP, enable OCSP stapling to reduce the overhead and latency of running SSL. ## Replace with your ssl_trusted_certificate. For more info see: diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 2eff1260b61..0230fbb010b 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -6,6 +6,7 @@ namespace :gitlab do desc "GITLAB | Create a backup of the GitLab system" task create: :environment do warn_user_is_not_gitlab + configure_cron_mode Rake::Task["gitlab:backup:db:create"].invoke Rake::Task["gitlab:backup:repo:create"].invoke @@ -21,6 +22,7 @@ namespace :gitlab do desc "GITLAB | Restore a previously created backup" task restore: :environment do warn_user_is_not_gitlab + configure_cron_mode backup = Backup::Manager.new backup.unpack @@ -35,43 +37,54 @@ namespace :gitlab do namespace :repo do task create: :environment do - puts "Dumping repositories ...".blue + $progress.puts "Dumping repositories ...".blue Backup::Repository.new.dump - puts "done".green + $progress.puts "done".green end task restore: :environment do - puts "Restoring repositories ...".blue + $progress.puts "Restoring repositories ...".blue Backup::Repository.new.restore - puts "done".green + $progress.puts "done".green end end namespace :db do task create: :environment do - puts "Dumping database ... ".blue + $progress.puts "Dumping database ... ".blue Backup::Database.new.dump - puts "done".green + $progress.puts "done".green end task restore: :environment do - puts "Restoring database ... ".blue + $progress.puts "Restoring database ... ".blue Backup::Database.new.restore - puts "done".green + $progress.puts "done".green end end namespace :uploads do task create: :environment do - puts "Dumping uploads ... ".blue + $progress.puts "Dumping uploads ... ".blue Backup::Uploads.new.dump - puts "done".green + $progress.puts "done".green end task restore: :environment do - puts "Restoring uploads ... ".blue + $progress.puts "Restoring uploads ... ".blue Backup::Uploads.new.restore - puts "done".green + $progress.puts "done".green + end + end + + def configure_cron_mode + if ENV['CRON'] + # We need an object we can say 'puts' and 'print' to; let's use a + # StringIO. + require 'stringio' + $progress = StringIO.new + else + $progress = $stdout end end end # namespace end: backup diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index f2705256f73..1da5f4b980f 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -574,20 +574,16 @@ namespace :gitlab do Gitlab::Shell.new.version end - def required_gitlab_shell_version - File.read(File.join(Rails.root, "GITLAB_SHELL_VERSION")).strip - end - def gitlab_shell_major_version - required_gitlab_shell_version.split(".")[0].to_i + Gitlab::Shell.version_required.split('.')[0].to_i end def gitlab_shell_minor_version - required_gitlab_shell_version.split(".")[1].to_i + Gitlab::Shell.version_required.split('.')[1].to_i end def gitlab_shell_patch_version - required_gitlab_shell_version.split(".")[2].to_i + Gitlab::Shell.version_required.split('.')[2].to_i end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 63dcdc52370..189ad6090a4 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -92,11 +92,11 @@ namespace :gitlab do User.ldap.each do |ldap_user| print "#{ldap_user.name} (#{ldap_user.extern_uid}) ..." - if Gitlab::LDAP::Access.open { |access| access.allowed?(ldap_user) } + if Gitlab::LDAP::Access.allowed?(ldap_user) puts " [OK]".green else if block_flag - ldap_user.block! + ldap_user.block! unless ldap_user.blocked? puts " [BLOCKED]".red else puts " [NOT IN LDAP]".yellow diff --git a/lib/tasks/gitlab/db/drop_all_postgres_sequences.rake b/lib/tasks/gitlab/db/drop_all_postgres_sequences.rake new file mode 100644 index 00000000000..e9cf0a9b5e8 --- /dev/null +++ b/lib/tasks/gitlab/db/drop_all_postgres_sequences.rake @@ -0,0 +1,10 @@ +namespace :gitlab do + namespace :db do + task drop_all_postgres_sequences: :environment do + connection = ActiveRecord::Base.connection + connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence| + connection.execute("DROP SEQUENCE #{sequence['relname']}") + end + end + end +end diff --git a/lib/tasks/gitlab/mail_google_schema_whitelisting.rake b/lib/tasks/gitlab/mail_google_schema_whitelisting.rake new file mode 100644 index 00000000000..f40bba24da8 --- /dev/null +++ b/lib/tasks/gitlab/mail_google_schema_whitelisting.rake @@ -0,0 +1,73 @@ +require "#{Rails.root}/app/helpers/emails_helper" +require 'action_view/helpers' +extend ActionView::Helpers + +include ActionView::Context +include EmailsHelper + +namespace :gitlab do + desc "Email google whitelisting email with example email for actions in inbox" + task mail_google_schema_whitelisting: :environment do + subject = "Rails | Implemented feature" + url = "#{Gitlab.config.gitlab.url}/base/rails-project/issues/#{rand(1..100)}#note_#{rand(10..1000)}" + schema = email_action(url) + body = email_template(schema, url) + mail = Notify.test_email("schema.whitelisting+sample@gmail.com", subject, body.html_safe) + if send_now + mail.deliver + else + puts "WOULD SEND:" + end + puts mail + end + + def email_template(schema, url) + "<html lang='en'> + <head> + <meta content='text/html; charset=utf-8' http-equiv='Content-Type'> + <title> + GitLab + </title> + </meta> + </head> + <style> + img { + max-width: 100%; + height: auto; + } + p.details { + font-style:italic; + color:#777 + } + .footer p { + font-size:small; + color:#777 + } + </style> + <body> + <div class='content'> + <div> + <p>I like it :+1: </p> + </div> + </div> + + <div class='footer' style='margin-top: 10px;'> + <p> + <br> + You're receiving this notification because you are a member of the Base / Rails Project project team. + <a href=\"#{url}\">View it on GitLab</a> + #{schema} + </p> + </div> + </body> + </html>" + end + + def send_now + if ENV['SEND'] == "true" + true + else + false + end + end +end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 55f338add6a..9af93300e08 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -4,7 +4,7 @@ namespace :gitlab do task :install, [:tag, :repo] => :environment do |t, args| warn_user_is_not_gitlab - default_version = File.read(File.join(Rails.root, "GITLAB_SHELL_VERSION")).strip + default_version = Gitlab::Shell.version_required args.with_defaults(tag: 'v' + default_version, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git") user = Gitlab.config.gitlab.user @@ -17,15 +17,19 @@ namespace :gitlab do # Clone if needed unless File.directory?(target_dir) - sh(*%W(git clone #{args.repo} #{target_dir})) + system(*%W(git clone -- #{args.repo} #{target_dir})) end # Make sure we're on the right tag Dir.chdir(target_dir) do # First try to checkout without fetching # to avoid stalling tests if the Internet is down. - reset = "git reset --hard $(git describe #{args.tag} || git describe origin/#{args.tag})" - sh "#{reset} || git fetch origin && #{reset}" + reseted = reset_to_commit(args) + + unless reseted + system(*%W(git fetch origin)) + reset_to_commit(args) + end config = { user: user, @@ -54,7 +58,7 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - sh "bin/install" + system(*%W(bin/install)) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -76,7 +80,7 @@ namespace :gitlab do desc "GITLAB | Build missing projects" task build_missing_projects: :environment do Project.find_each(batch_size: 1000) do |project| - path_to_repo = File.join(Gitlab.config.gitlab_shell.repos_path, "#{project.path_with_namespace}.git") + path_to_repo = project.repository.path_to_repo if File.exists?(path_to_repo) print '-' else @@ -118,5 +122,16 @@ namespace :gitlab do puts "Quitting...".red exit 1 end + + def reset_to_commit(args) + tag, status = Gitlab::Popen.popen(%W(git describe -- #{args.tag})) + + unless status.zero? + tag, status = Gitlab::Popen.popen(%W(git describe -- origin/#{args.tag})) + end + + tag = tag.strip + system(*%W(git reset --hard #{tag})) + end end |