From be414d4829463708e23b639fea9e6fe88863b1c5 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Thu, 1 Sep 2016 02:21:16 +0800 Subject: We don't need them --- lib/tasks/.gitkeep | 0 lib/tasks/ci/.gitkeep | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 lib/tasks/.gitkeep delete mode 100644 lib/tasks/ci/.gitkeep (limited to 'lib/tasks') diff --git a/lib/tasks/.gitkeep b/lib/tasks/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 -- cgit v1.2.1 From dffd33252f029901e33883935b20f6b0368d819b Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Sun, 25 Sep 2016 11:55:14 +0200 Subject: Move reply by email docs to a new location [ci skip] --- lib/tasks/gitlab/check.rake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 5f4a6bbfa35..2ae48a970ce 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -671,7 +671,7 @@ namespace :gitlab do "Enable mail_room in the init.d configuration." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -690,7 +690,7 @@ namespace :gitlab do "Enable mail_room in your Procfile." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -747,7 +747,7 @@ namespace :gitlab do "Check that the information in config/gitlab.yml is correct" ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end -- cgit v1.2.1 From e3f12e975d59b0606b8f1365e21814b34ea83767 Mon Sep 17 00:00:00 2001 From: Connor Shea Date: Wed, 28 Sep 2016 19:38:43 -0600 Subject: Remove Flog as we use a Rubocop that does its job. --- lib/tasks/flog.rake | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 lib/tasks/flog.rake (limited to 'lib/tasks') diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake deleted file mode 100644 index 3bfe999ae74..00000000000 --- a/lib/tasks/flog.rake +++ /dev/null @@ -1,25 +0,0 @@ -desc 'Code complexity analyze via flog' -task :flog do - output = %x(bundle exec flog -m app/ lib/gitlab) - exit_code = 0 - minimum_score = 70 - output = output.lines - - # Skip total complexity score - output.shift - - # Skip some trash info - output.shift - - output.each do |line| - score, method = line.split(" ") - score = score.to_i - - if score > minimum_score - exit_code = 1 - puts "High complexity in #{method}. Score: #{score}" - end - end - - exit exit_code -end -- cgit v1.2.1 From fe46e4eb35dd6728a8a5ebcee9cc05a4613effbf Mon Sep 17 00:00:00 2001 From: Justin DiPierro Date: Thu, 29 Sep 2016 12:46:54 -0400 Subject: Load Github::Shell's secret token from file on initialization instead of every request. --- lib/tasks/gitlab/shell.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index bb7eb852f1b..210899882b4 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -78,7 +78,7 @@ namespace :gitlab do f.puts "PATH=#{ENV['PATH']}" end - Gitlab::Shell.new.generate_and_link_secret_token + Gitlab::Shell.ensure_secret_token! end desc "GitLab | Setup gitlab-shell" -- cgit v1.2.1 From e94cd6fdfe43d9128d37a539cf84f4388c5cf970 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 6 Oct 2016 22:17:11 +0100 Subject: Add markdown cache columns to the database, but don't use them yet This commit adds a number of _html columns and, with the exception of Note, starts updating them whenever the content of their partner fields changes. Note has a collision with the note_html attr_accessor; that will be fixed later A background worker for clearing these cache columns is also introduced - use `rake cache:clear` to set it off. You can clear the database or Redis caches separately by running `rake cache:clear:db` or `rake cache:clear:redis`, respectively. --- lib/tasks/cache.rake | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) (limited to 'lib/tasks') diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 2214f855200..a95a3455a4a 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,22 +1,33 @@ namespace :cache do - CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan + namespace :clear do + REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 + REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan - desc "GitLab | Clear redis cache" - task :clear => :environment do - Gitlab::Redis.with do |redis| - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + desc "GitLab | Clear redis cache" + task redis: :environment do + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: REDIS_CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end + + desc "GitLab | Clear database cache (in the background)" + task db: :environment do + ClearDatabaseCacheWorker.perform_async + end + + task all: [:db, :redis] end + + task clear: 'cache:clear:all' end -- cgit v1.2.1 From ebba49149395ba6fb0f14c3aa9c46a496b234dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 7 Oct 2016 18:35:36 +0200 Subject: Add a new gitlab:users:clear_all_authentication_tokens task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/tasks/gitlab/users.rake | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 lib/tasks/gitlab/users.rake (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake new file mode 100644 index 00000000000..3a16ace60bd --- /dev/null +++ b/lib/tasks/gitlab/users.rake @@ -0,0 +1,11 @@ +namespace :gitlab do + namespace :users do + desc "GitLab | Clear the authentication token for all users" + task clear_all_authentication_tokens: :environment do |t, args| + # Do small batched updates because these updates will be slow and locking + User.select(:id).find_in_batches(batch_size: 100) do |batch| + User.where(id: batch.map(&:id)).update_all(authentication_token: nil) + end + end + end +end -- cgit v1.2.1 From 069f2d347585a0f79ab8e3ddfb194ebbc86176c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 29 Sep 2016 19:31:14 +0200 Subject: Draft a quick CE->EE merge check rake task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/tasks/ce_to_ee_merge_check.rake | 6 +++ lib/tasks/gitlab/dev.rake | 96 +++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 lib/tasks/ce_to_ee_merge_check.rake create mode 100644 lib/tasks/gitlab/dev.rake (limited to 'lib/tasks') diff --git a/lib/tasks/ce_to_ee_merge_check.rake b/lib/tasks/ce_to_ee_merge_check.rake new file mode 100644 index 00000000000..bb87f85cd9c --- /dev/null +++ b/lib/tasks/ce_to_ee_merge_check.rake @@ -0,0 +1,6 @@ +desc 'Checks if the branch would apply cleanly to EE' +task ce_to_ee_merge_check: :environment do + return if defined?(Gitlab::License) + + Rake::Task['gitlab:dev:ce_to_ee_merge_check'].invoke +end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake new file mode 100644 index 00000000000..bf17ba499bc --- /dev/null +++ b/lib/tasks/gitlab/dev.rake @@ -0,0 +1,96 @@ +namespace :gitlab do + namespace :dev do + desc 'Checks if the branch would apply cleanly to EE' + task ce_to_ee_merge_check: :environment do + ce_repo = ENV['CI_BUILD_REPO'] + ce_branch = ENV['CI_BUILD_REF_NAME'] + + ee_repo = 'https://gitlab.com/gitlab-org/gitlab-ee.git' + ee_branch = "#{ce_branch}-ee" + ee_dir = 'gitlab-ee-merge-check' + + puts "\n=> Cloning #{ee_repo} into #{ee_dir}\n" + `git clone #{ee_repo} #{ee_dir} --depth 1` + Dir.chdir(ee_dir) do + puts "\n => Fetching #{ce_repo}/#{ce_branch}\n" + `git fetch #{ce_repo} #{ce_branch} --depth 1` + + # Try to merge the current tested branch to EE/master... + puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" + `git merge --ff-only FETCH_HEAD` + + exit 0 if $?.success? + + # Try to merge a possible -ee branch to EE/master... + puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n" + `git merge --ff-only #{ee_branch}` + + # The -ee doesn't exist + if $?.exitstatus == 1 + puts <<-MSG.strip_heredoc + \n================================================================= + The #{ce_branch} branch cannot be merged without conflicts to the + current EE/master, and no #{ee_branch} branch was detected in + the EE repository. + + Please create a #{ee_branch} branch that includes changes + #{ce_branch} but also specific changes than can be applied cleanly + to EE/master. + + You can create this branch as follow: + + 1. In the EE repo: + $ git fetch origin + $ git fetch #{ce_repo} #{ce_branch} + $ git checkout -b #{ee_branch} FETCH_HEAD + $ git rebase origin/master + 2. At this point you will likely have conflicts, solve them, and + continue/finish the rebase. + 3. You can squash all the original #{ce_branch} commits into a + single "Port of #{ce_branch} to EE". + 4. Push your branch to #{ee_repo}: + $ git push origin #{ee_branch} + =================================================================\n + MSG + + exit 1 + end + + # The -ee cannot be merged cleanly to EE/master... + unless $?.success? + puts <<-MSG.strip_heredoc + \n================================================================= + The #{ce_branch} branch cannot be merged without conflicts to + EE/master, and even though the #{ee_branch} branch exists in the EE + repository, it cannot be merged without conflicts to EE/master. + + Please update the #{ee_branch}, push it again to #{ee_repo}, and + retry this job. + =================================================================\n + MSG + + exit 2 + end + + puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" + `git merge --ff-only FETCH_HEAD` + exit 0 if $?.success? + + # The -ee can be merged cleanly to EE/master, but still + # cannot be merged cleanly to EE/master... + puts <<-MSG.strip_heredoc + \n================================================================= + The #{ce_branch} branch cannot be merged without conflicts to EE, and + even though the #{ee_branch} branch exists in the EE repository and + applies cleanly to EE/master, it doesn't prevent conflicts when + merging #{ce_branch} into EE. + + We may be in a complex situation here. + =================================================================\n + MSG + + exit 3 + end + end + end +end -- cgit v1.2.1 From 2650d5f895c6f7e56f7ba76e5fa448d38fd15d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 7 Oct 2016 17:21:51 +0200 Subject: Improve the branch existence and merge checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also add a safeguard for non-CI env. Signed-off-by: Rémy Coutable --- lib/tasks/ce_to_ee_merge_check.rake | 2 -- lib/tasks/gitlab/dev.rake | 35 +++++++++++++++++++++++------------ 2 files changed, 23 insertions(+), 14 deletions(-) (limited to 'lib/tasks') diff --git a/lib/tasks/ce_to_ee_merge_check.rake b/lib/tasks/ce_to_ee_merge_check.rake index bb87f85cd9c..424e7883060 100644 --- a/lib/tasks/ce_to_ee_merge_check.rake +++ b/lib/tasks/ce_to_ee_merge_check.rake @@ -1,6 +1,4 @@ desc 'Checks if the branch would apply cleanly to EE' task ce_to_ee_merge_check: :environment do - return if defined?(Gitlab::License) - Rake::Task['gitlab:dev:ce_to_ee_merge_check'].invoke end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index bf17ba499bc..47bdb2d32d2 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -2,6 +2,9 @@ namespace :gitlab do namespace :dev do desc 'Checks if the branch would apply cleanly to EE' task ce_to_ee_merge_check: :environment do + return if defined?(Gitlab::License) + return unless ENV['CI'] + ce_repo = ENV['CI_BUILD_REPO'] ce_branch = ENV['CI_BUILD_REF_NAME'] @@ -17,27 +20,28 @@ namespace :gitlab do # Try to merge the current tested branch to EE/master... puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" - `git merge --ff-only FETCH_HEAD` + `git merge FETCH_HEAD` exit 0 if $?.success? - # Try to merge a possible -ee branch to EE/master... - puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n" - `git merge --ff-only #{ee_branch}` + # Check if the -ee branch exists... + puts "\n => Check if #{ee_repo}/#{ee_branch} exists\n" + `git rev-parse --verify #{ee_branch}` # The -ee doesn't exist - if $?.exitstatus == 1 + unless $?.success? + puts puts <<-MSG.strip_heredoc - \n================================================================= + ================================================================= The #{ce_branch} branch cannot be merged without conflicts to the current EE/master, and no #{ee_branch} branch was detected in the EE repository. - Please create a #{ee_branch} branch that includes changes + Please create a #{ee_branch} branch that includes changes from #{ce_branch} but also specific changes than can be applied cleanly to EE/master. - You can create this branch as follow: + You can create this branch as follows: 1. In the EE repo: $ git fetch origin @@ -45,7 +49,8 @@ namespace :gitlab do $ git checkout -b #{ee_branch} FETCH_HEAD $ git rebase origin/master 2. At this point you will likely have conflicts, solve them, and - continue/finish the rebase. + continue/finish the rebase. Note: You can squash the CE commits + before rebasing. 3. You can squash all the original #{ce_branch} commits into a single "Port of #{ce_branch} to EE". 4. Push your branch to #{ee_repo}: @@ -56,10 +61,15 @@ namespace :gitlab do exit 1 end + # Try to merge the -ee branch to EE/master... + puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n" + `git merge #{ee_branch} master` + # The -ee cannot be merged cleanly to EE/master... unless $?.success? + puts puts <<-MSG.strip_heredoc - \n================================================================= + ================================================================= The #{ce_branch} branch cannot be merged without conflicts to EE/master, and even though the #{ee_branch} branch exists in the EE repository, it cannot be merged without conflicts to EE/master. @@ -73,13 +83,14 @@ namespace :gitlab do end puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" - `git merge --ff-only FETCH_HEAD` + `git merge FETCH_HEAD` exit 0 if $?.success? # The -ee can be merged cleanly to EE/master, but still # cannot be merged cleanly to EE/master... + puts puts <<-MSG.strip_heredoc - \n================================================================= + ================================================================= The #{ce_branch} branch cannot be merged without conflicts to EE, and even though the #{ee_branch} branch exists in the EE repository and applies cleanly to EE/master, it doesn't prevent conflicts when -- cgit v1.2.1 From 0c0caede85b0ea6082799ae9fa6afd74a1186006 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Thu, 20 Oct 2016 17:33:12 +0300 Subject: Fix: Backup restore doesn't clear cache --- lib/tasks/gitlab/backup.rake | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/tasks') diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index b43ee5b3383..a9f1255e8cf 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -51,6 +51,7 @@ namespace :gitlab do $progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') @@ -58,6 +59,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke backup.cleanup end -- cgit v1.2.1 From c0eb2cbbb4b14e3abb843a65d685bee400b5fffe Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Sat, 22 Oct 2016 12:42:19 +0100 Subject: Stop clearing the database cache on rake cache:clear --- lib/tasks/cache.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/tasks') diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index a95a3455a4a..78ae187817a 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -29,5 +29,5 @@ namespace :cache do task all: [:db, :redis] end - task clear: 'cache:clear:all' + task clear: 'cache:clear:redis' end -- cgit v1.2.1