From 7a8a892efdf59925a95cdf6504f7c74c31b87eeb Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 25 Sep 2015 17:12:41 +0200 Subject: Add "rake gitlab:list_repos" task --- lib/tasks/gitlab/list_repos.rake | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 lib/tasks/gitlab/list_repos.rake (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake new file mode 100644 index 00000000000..1377e1ea910 --- /dev/null +++ b/lib/tasks/gitlab/list_repos.rake @@ -0,0 +1,16 @@ +namespace :gitlab do + task list_repos: :environment do + scope = Project + if ENV['SINCE'] + date = Time.parse(ENV['SINCE']) + warn "Listing repositories with activity since #{date}" + project_ids = Project.where(['last_activity_at > ?', date]).pluck(:id) + scope = scope.where(id: project_ids) + end + scope.find_each do |project| + base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + puts base + '.git' + puts base + '.wiki.git' + end + end +end -- cgit v1.2.1 From 54e6c0045bb13c05cc5478cbdf47d3246bd9fe2b Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 22 Sep 2015 14:04:14 +0200 Subject: Added three rake tasks for repository maintainance Tasks added: gitlab:git:repack gitlab:git:gc gitlab:git:prune --- lib/tasks/gitlab/git.rake | 52 ++++++++++++++++++++++++++++++++++++++ lib/tasks/gitlab/task_helpers.rake | 4 +++ 2 files changed, 56 insertions(+) create mode 100644 lib/tasks/gitlab/git.rake (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake new file mode 100644 index 00000000000..4fbf5a9393c --- /dev/null +++ b/lib/tasks/gitlab/git.rake @@ -0,0 +1,52 @@ +namespace :gitlab do + namespace :git do + + desc "GitLab | Git | Repack" + task repack: :environment do + failures = perform_git_cmd('git repack -a --quiet', 'Git repack') + if failures.empty? + puts "Done".green + else + output_failures(failures) + end + end + + desc "GitLab | Git | Run gits garbage collection on all repo's" + task gc: :environment do + failures = perform_git_cmd('git gc --auto --quiet', "Garbage Collection") + if failures.empty? + puts "Done".green + else + output_failures(failures) + end + end + + desc "GitLab | Git | Git prune all repo's" + task prune: :environment do + failures = perform_git_cmd('git prune', 'Git Prune') + if failures.empty? + puts "Done".green + else + output_failures(failures) + end + end + + def perform_git_cmd(cmd, message) + puts "Starting #{message} on all repositories" + + failures = [] + all_repos.each do |r| + puts "Performing #{message} at #{r}" + failures << r unless system(*%w(#{cmd}), chdir: r) + end + + failures + end + + def output_failures(failures) + puts "The following repositories reported errors:".red + failures.each { |f| puts "- #{f}" } + end + + end +end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index c95b6540ebc..77c41bf61cb 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -128,4 +128,8 @@ namespace :gitlab do false end end + + def all_repos + Dir.glob(File.join(Gitlab.config.gitlab_shell.repos_path, '**/*\.git')) + end end -- cgit v1.2.1 From 8db063b579322238af43f6d04b5968d9c6ea935d Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 1 Oct 2015 13:34:41 +0200 Subject: Scalable way of requesting all repos --- lib/tasks/gitlab/git.rake | 19 +++++++++++-------- lib/tasks/gitlab/task_helpers.rake | 6 +++++- 2 files changed, 16 insertions(+), 9 deletions(-) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index 4fbf5a9393c..65ee430d550 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -3,7 +3,7 @@ namespace :gitlab do desc "GitLab | Git | Repack" task repack: :environment do - failures = perform_git_cmd('git repack -a --quiet', 'Git repack') + failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo") if failures.empty? puts "Done".green else @@ -11,9 +11,9 @@ namespace :gitlab do end end - desc "GitLab | Git | Run gits garbage collection on all repo's" + desc "GitLab | Git | Run garbage collection on all repos" task gc: :environment do - failures = perform_git_cmd('git gc --auto --quiet', "Garbage Collection") + failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting") if failures.empty? puts "Done".green else @@ -21,9 +21,9 @@ namespace :gitlab do end end - desc "GitLab | Git | Git prune all repo's" + desc "GitLab | Git | Prune all repos" task prune: :environment do - failures = perform_git_cmd('git prune', 'Git Prune') + failures = perform_git_cmd(%W(git prune), "Git Prune") if failures.empty? puts "Done".green else @@ -35,9 +35,12 @@ namespace :gitlab do puts "Starting #{message} on all repositories" failures = [] - all_repos.each do |r| - puts "Performing #{message} at #{r}" - failures << r unless system(*%w(#{cmd}), chdir: r) + all_repos do |repo| + if system(*cmd, chdir: repo) + puts "Performed #{message} at #{repo}" + else + failures << repo + end end failures diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index 77c41bf61cb..e35fd47c5c4 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -130,6 +130,10 @@ namespace :gitlab do end def all_repos - Dir.glob(File.join(Gitlab.config.gitlab_shell.repos_path, '**/*\.git')) + IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end + end end end -- cgit v1.2.1 From 3cc2b48a82aae8b535ddf11e9057a29f2242524c Mon Sep 17 00:00:00 2001 From: Marin Jankovski Date: Wed, 18 Nov 2015 11:39:26 +0100 Subject: Backup LFS objects same as any upload. --- lib/tasks/gitlab/backup.rake | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 3c46bcea40e..cb4abe13799 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -13,6 +13,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke + Rake::Task["gitlab:backup:lfs:create"].invoke backup = Backup::Manager.new backup.pack @@ -34,6 +35,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") + Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -134,6 +136,25 @@ namespace :gitlab do end end + namespace :lfs do + task create: :environment do + $progress.puts "Dumping lfs objects ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("lfs") + $progress.puts "[SKIPPED]".cyan + else + Backup::Lfs.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring lfs objects ... ".blue + Backup::Lfs.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a -- cgit v1.2.1 From ec002e802677a691d758ebc6f94f23020b2b063a Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 24 Nov 2015 19:20:23 -0500 Subject: Remove usage of Colored --- lib/tasks/gitlab/task_helpers.rake | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index c95b6540ebc..efb863a8764 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,16 +2,6 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end -unless STDOUT.isatty - module Colored - extend self - - def colorize(string, options={}) - string - end - end -end - namespace :gitlab do # Ask if the user wants to continue @@ -103,7 +93,7 @@ namespace :gitlab do gitlab_user = Gitlab.config.gitlab.user current_user = run(%W(whoami)).chomp unless current_user == gitlab_user - puts "#{Colored.color(:black)+Colored.color(:on_yellow)} Warning #{Colored.extra(:clear)}" + puts " Warning ".colorize(:black).on_yellow puts " You are running as user #{current_user.magenta}, we hope you know what you are doing." puts " Things may work\/fail for the wrong reasons." puts " For correct results you should run this as user #{gitlab_user.magenta}." -- cgit v1.2.1 From 23f383ef69889c9829ad36afa53b5abfbf4b5511 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 8 Dec 2015 16:06:06 +0100 Subject: Detect project and namespace changes in list:repos --- lib/tasks/gitlab/list_repos.rake | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index 1377e1ea910..c7596e7abcb 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -3,9 +3,10 @@ namespace :gitlab do scope = Project if ENV['SINCE'] date = Time.parse(ENV['SINCE']) - warn "Listing repositories with activity since #{date}" - project_ids = Project.where(['last_activity_at > ?', date]).pluck(:id) - scope = scope.where(id: project_ids) + warn "Listing repositories with activity or changes since #{date}" + project_ids = Project.where('last_activity_at > ? OR updated_at > ?', date, date).pluck(:id).sort + namespace_ids = Namespace.where(['updated_at > ?', date]).pluck(:id).sort + scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) -- cgit v1.2.1 From 1d410ac96a96462d9f48ec4e43a4e819bbffdeee Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 7 Dec 2015 23:01:53 -0800 Subject: Update project repository size and commit count during import:repos task Closes #3848 --- lib/tasks/gitlab/import.rake | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index c1ee271ae2b..1c04f47f08f 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -64,6 +64,8 @@ namespace :gitlab do if project.persisted? puts " * Created #{project.name} (#{repo_path})".green + project.update_repository_size + project.update_commit_count else puts " * Failed trying to create #{project.name} (#{repo_path})".red puts " Errors: #{project.errors.messages}".red -- cgit v1.2.1 From 9ebdee09673d79b57c07c7332ee87791229b6f72 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 9 Dec 2015 10:50:38 +0100 Subject: Split up feature specs more --- lib/tasks/spinach.rake | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'lib/tasks') diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index d5a96fd38f4..3acfc6e2075 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -1,11 +1,31 @@ Rake::Task["spinach"].clear if Rake::Task.task_defined?('spinach') namespace :spinach do + namespace :project do + desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features" + task :half do + cmds = [ + %W(rake gitlab:setup), + %W(spinach --tags @project_commits,@project_issues,@project_merge_requests), + ] + run_commands(cmds) + end + + desc "GitLab | Spinach | Run remaining project spinach features" + task :rest do + cmds = [ + %W(rake gitlab:setup), + %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests), + ] + run_commands(cmds) + end + end + desc "GitLab | Spinach | Run project spinach features" task :project do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits), + %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets), ] run_commands(cmds) end @@ -14,7 +34,7 @@ namespace :spinach do task :other do cmds = [ %W(rake gitlab:setup), - %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits), + %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets), ] run_commands(cmds) end @@ -33,4 +53,4 @@ def run_commands(cmds) cmds.each do |cmd| system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") end -end \ No newline at end of file +end -- cgit v1.2.1 From ae0b9017315b38dd42b4a1c9b6fb1daa78fee28a Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 9 Dec 2015 10:51:01 +0100 Subject: Split up specs more --- lib/tasks/spec.rake | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib/tasks') diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 365ff2defd4..049e17e6c11 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -19,6 +19,15 @@ namespace :spec do run_commands(cmds) end + desc 'GitLab | Rspec | Run model specs' + task :models do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @models) + ] + run_commands(cmds) + end + desc 'GitLab | Rspec | Run benchmark specs' task :benchmark do cmds = [ @@ -32,7 +41,7 @@ namespace :spec do task :other do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark) + %W(rspec spec --tag ~@api,~@feature,~@models,~@benchmark) ] run_commands(cmds) end -- cgit v1.2.1 From 7a5e77c0a0591ff5d121f93deced27b798ddbee3 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 9 Dec 2015 11:09:25 +0100 Subject: Fix rspec tag syntax --- lib/tasks/spec.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/tasks') diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 049e17e6c11..343e4b63524 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -41,7 +41,7 @@ namespace :spec do task :other do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@api,~@feature,~@models,~@benchmark) + %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@benchmark) ] run_commands(cmds) end -- cgit v1.2.1 From 6c6fb1d8bb2c6aa44553ebf35496d64ff6b202d2 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 9 Dec 2015 11:56:23 +0100 Subject: Split up spec:other even more --- lib/tasks/spec.rake | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'lib/tasks') diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 343e4b63524..0985ef3a669 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -28,6 +28,24 @@ namespace :spec do run_commands(cmds) end + desc 'GitLab | Rspec | Run service specs' + task :services do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @services) + ] + run_commands(cmds) + end + + desc 'GitLab | Rspec | Run lib specs' + task :lib do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @lib) + ] + run_commands(cmds) + end + desc 'GitLab | Rspec | Run benchmark specs' task :benchmark do cmds = [ @@ -41,7 +59,7 @@ namespace :spec do task :other do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@benchmark) + %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services --tag ~@benchmark) ] run_commands(cmds) end -- cgit v1.2.1 From e80e3f5372d6bcad1fbe04a85b3086bb66794828 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 4 Dec 2015 12:55:23 +0100 Subject: Migrate CI::Project to Project --- lib/tasks/ci/schedule_builds.rake | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 lib/tasks/ci/schedule_builds.rake (limited to 'lib/tasks') diff --git a/lib/tasks/ci/schedule_builds.rake b/lib/tasks/ci/schedule_builds.rake deleted file mode 100644 index 49435504c67..00000000000 --- a/lib/tasks/ci/schedule_builds.rake +++ /dev/null @@ -1,6 +0,0 @@ -namespace :ci do - desc "GitLab CI | Clean running builds" - task schedule_builds: :environment do - Ci::Scheduler.new.perform - end -end -- cgit v1.2.1 From 9c5d8079a367ac24d04466f03f6b9abf5c333f59 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 24 Nov 2015 08:28:18 -0800 Subject: Bump Redis requirement to 2.8 for Sidekiq 4 requirements Closes #3649 [ci skip] --- lib/tasks/gitlab/check.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a25fac62cfc..b5af3d88b4c 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -331,7 +331,7 @@ namespace :gitlab do end def check_redis_version - min_redis_version = "2.4.0" + min_redis_version = "2.8.0" print "Redis version >= #{min_redis_version}? ... " redis_version = run(%W(redis-cli --version)) -- cgit v1.2.1 From f8bf6c4b2c5fb41ae92df09b9f968d5d5d61bb2b Mon Sep 17 00:00:00 2001 From: Drew Blessing Date: Fri, 11 Dec 2015 17:27:20 -0600 Subject: [ci skip] Add user repository integrity check rake task --- lib/tasks/gitlab/check.rake | 56 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a25fac62cfc..167b0112666 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -822,10 +822,27 @@ namespace :gitlab do namespace_dirs.each do |namespace_dir| repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each do |dir| - puts "\nChecking repo at #{dir}" - system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir) - end + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end + end + end + + namespace :user do + desc "GitLab | Check the integrity of a specific user's repositories" + task :check_repos, [:username] => :environment do |t, args| + username = args[:username] || prompt("Check repository integrity for which username? ".blue) + user = User.find_by(username: username) + if user + repo_dirs = user.authorized_projects.map do |p| + File.join( + Gitlab.config.gitlab_shell.repos_path, + "#{p.path_with_namespace}.git" + ) + end + + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + else + puts "\nUser '#{username}' not found".red end end end @@ -952,4 +969,35 @@ namespace :gitlab do false end end + + def check_repo_integrity(repo_dir) + puts "\nChecking repo at #{repo_dir.yellow}" + + git_fsck(repo_dir) + check_config_lock(repo_dir) + check_ref_locks(repo_dir) + end + + def git_fsck(repo_dir) + puts "Running `git fsck`".yellow + system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir) + end + + def check_config_lock(repo_dir) + config_exists = File.exist?(File.join(repo_dir,'config.lock')) + config_output = config_exists ? 'yes'.red : 'no'.green + puts "'config.lock' file exists?".yellow + " ... #{config_output}" + end + + def check_ref_locks(repo_dir) + lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock')) + if lock_files.present? + puts "Ref lock files exist:".red + lock_files.each do |lock_file| + puts " #{lock_file}" + end + else + puts "No ref lock files exist".green + end + end end -- cgit v1.2.1