diff options
Diffstat (limited to 'lib/tasks')
| -rw-r--r-- | lib/tasks/gitlab/assets.rake | 2 | ||||
| -rw-r--r-- | lib/tasks/gitlab/dev.rake | 6 | ||||
| -rw-r--r-- | lib/tasks/gitlab/gitaly.rake | 7 | ||||
| -rw-r--r-- | lib/tasks/gitlab/shell.rake | 2 | ||||
| -rw-r--r-- | lib/tasks/gitlab/storage.rake | 85 | ||||
| -rw-r--r-- | lib/tasks/gitlab/users.rake | 11 | ||||
| -rw-r--r-- | lib/tasks/import.rake | 27 | ||||
| -rw-r--r-- | lib/tasks/tokens.rake | 12 |
8 files changed, 119 insertions, 33 deletions
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 259a755d724..a42f02a84fd 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -3,8 +3,8 @@ namespace :gitlab do desc 'GitLab | Assets | Compile all frontend assets' task compile: [ 'yarn:check', - 'rake:assets:precompile', 'gettext:po_to_json', + 'rake:assets:precompile', 'webpack:compile', 'fix_urls' ] diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 3eade7bf553..ba221e44e5d 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -4,7 +4,11 @@ namespace :gitlab do task :ee_compat_check, [:branch] => :environment do |_, args| opts = if ENV['CI'] - { branch: ENV['CI_COMMIT_REF_NAME'] } + { + ce_project_url: ENV['CI_PROJECT_URL'], + branch: ENV['CI_COMMIT_REF_NAME'], + job_id: ENV['CI_JOB_ID'] + } else unless args[:branch] puts "Must specify a branch as an argument".color(:red) diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 08677a98fc1..8377fe3269d 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -50,6 +50,8 @@ namespace :gitlab do # only generate a configuration for the most common and simplest case: when # we have exactly one Gitaly process and we are sure it is running locally # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. def gitaly_configuration_toml(gitaly_ruby: true) storages = [] address = nil @@ -67,6 +69,11 @@ namespace :gitlab do storages << { name: key, path: val['path'] } end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 42825f29e32..0e6aed32c52 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -79,7 +79,7 @@ namespace :gitlab do if File.exist?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.repository_storage_path, + if Gitlab::Shell.new.add_repository(project.repository_storage, project.disk_path) print '.' else diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake new file mode 100644 index 00000000000..e05be4a3405 --- /dev/null +++ b/lib/tasks/gitlab/storage.rake @@ -0,0 +1,85 @@ +namespace :gitlab do + namespace :storage do + desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' + task migrate_to_hashed: :environment do + legacy_projects_count = Project.with_legacy_storage.count + + if legacy_projects_count == 0 + puts 'There are no projects using legacy storage. Nothing to do!' + + next + end + + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + + project_id_batches do |start, finish| + StorageMigratorWorker.perform_async(start, finish) + + print '.' + end + + puts ' Done!' + end + + desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' + task legacy_projects: :environment do + projects_summary(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | List existing projects using Legacy Storage' + task list_legacy_projects: :environment do + projects_list(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' + task hashed_projects: :environment do + projects_summary(Project.with_hashed_storage) + end + + desc 'Gitlab | Storage | List existing projects using Hashed Storage' + task list_hashed_projects: :environment do + projects_list(Project.with_hashed_storage) + end + + def batch_size + ENV.fetch('BATCH', 200).to_i + end + + def project_id_batches(&block) + Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + + def projects_summary(relation) + projects_count = relation.count + puts "* Found #{projects_count} projects".color(:green) + + projects_count + end + + def projects_list(relation) + projects_count = projects_summary(relation) + + projects = relation.with_route + limit = ENV.fetch('LIMIT', 500).to_i + + return unless projects_count > 0 + + puts " ! Displaying first #{limit} projects..." if projects_count > limit + + counter = 0 + projects.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |project| + counter += 1 + + puts " - #{project.full_path} (id: #{project.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end + end +end diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake deleted file mode 100644 index 3a16ace60bd..00000000000 --- a/lib/tasks/gitlab/users.rake +++ /dev/null @@ -1,11 +0,0 @@ -namespace :gitlab do - namespace :users do - desc "GitLab | Clear the authentication token for all users" - task clear_all_authentication_tokens: :environment do |t, args| - # Do small batched updates because these updates will be slow and locking - User.select(:id).find_in_batches(batch_size: 100) do |batch| - User.where(id: batch.map(&:id)).update_all(authentication_token: nil) - end - end - end -end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 4d485108cf6..7f86fd7b45e 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -39,13 +39,19 @@ class GithubImport def import! @project.force_import_start + import_success = false + timings = Benchmark.measure do - Github::Import.new(@project, @options).execute + import_success = Github::Import.new(@project, @options).execute end - puts "Import finished. Timings: #{timings}".color(:green) - - @project.import_finish + if import_success + @project.import_finish + puts "Import finished. Timings: #{timings}".color(:green) + else + puts "Import was not successful. Errors were as follows:" + puts @project.import_error + end end def new_project @@ -53,18 +59,23 @@ class GithubImport namespace_path, _sep, name = @project_path.rpartition('/') namespace = find_or_create_namespace(namespace_path) - Projects::CreateService.new( + project = Projects::CreateService.new( @current_user, name: name, path: name, description: @repo['description'], namespace_id: namespace.id, visibility_level: visibility_level, - import_type: 'github', - import_source: @repo['full_name'], - import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"), skip_wiki: @repo['has_wiki'] ).execute + + project.update!( + import_type: 'github', + import_source: @repo['full_name'], + import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@") + ) + + project end end diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index ad1818ff1fa..693597afdf8 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -1,12 +1,7 @@ require_relative '../../app/models/concerns/token_authenticatable.rb' namespace :tokens do - desc "Reset all GitLab user auth tokens" - task reset_all_auth: :environment do - reset_all_users_token(:reset_authentication_token!) - end - - desc "Reset all GitLab email tokens" + desc "Reset all GitLab incoming email tokens" task reset_all_email: :environment do reset_all_users_token(:reset_incoming_email_token!) end @@ -31,11 +26,6 @@ class TmpUser < ActiveRecord::Base self.table_name = 'users' - def reset_authentication_token! - write_new_token(:authentication_token) - save!(validate: false) - end - def reset_incoming_email_token! write_new_token(:incoming_email_token) save!(validate: false) |