summaryrefslogtreecommitdiff
path: root/lib/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'lib/tasks')
-rw-r--r--lib/tasks/gitlab/artifacts/migrate.rake25
-rw-r--r--lib/tasks/gitlab/check.rake8
-rw-r--r--lib/tasks/gitlab/cleanup.rake4
-rw-r--r--lib/tasks/gitlab/info.rake2
-rw-r--r--lib/tasks/gitlab/lfs/migrate.rake22
-rw-r--r--lib/tasks/gitlab/two_factor.rake2
-rw-r--r--lib/tasks/gitlab/uploads/migrate.rake34
-rw-r--r--lib/tasks/migrate/setup_postgresql.rake4
-rw-r--r--lib/tasks/test.rake5
9 files changed, 91 insertions, 15 deletions
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
new file mode 100644
index 00000000000..bfca4bfb3f7
--- /dev/null
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -0,0 +1,25 @@
+require 'logger'
+require 'resolv-replace'
+
+desc "GitLab | Migrate files for artifacts to comply with new storage format"
+namespace :gitlab do
+ namespace :artifacts do
+ task migrate: :environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Starting transfer of artifacts')
+
+ Ci::Build.joins(:project)
+ .with_artifacts_stored_locally
+ .find_each(batch_size: 10) do |build|
+ begin
+ build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
+ build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
+
+ logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
+ rescue => e
+ logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+ end
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 2403f57f05a..abef8cd2bcc 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -61,7 +61,7 @@ namespace :gitlab do
puts "Repo base directory exists?"
Gitlab.config.repositories.storages.each do |name, repository_storage|
- repo_base_path = repository_storage['path']
+ repo_base_path = repository_storage.legacy_disk_path
print "#{name}... "
if File.exist?(repo_base_path)
@@ -86,7 +86,7 @@ namespace :gitlab do
puts "Repo storage directories are symlinks?"
Gitlab.config.repositories.storages.each do |name, repository_storage|
- repo_base_path = repository_storage['path']
+ repo_base_path = repository_storage.legacy_disk_path
print "#{name}... "
unless File.exist?(repo_base_path)
@@ -110,7 +110,7 @@ namespace :gitlab do
puts "Repo paths access is drwxrws---?"
Gitlab.config.repositories.storages.each do |name, repository_storage|
- repo_base_path = repository_storage['path']
+ repo_base_path = repository_storage.legacy_disk_path
print "#{name}... "
unless File.exist?(repo_base_path)
@@ -140,7 +140,7 @@ namespace :gitlab do
puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?"
Gitlab.config.repositories.storages.each do |name, repository_storage|
- repo_base_path = repository_storage['path']
+ repo_base_path = repository_storage.legacy_disk_path
print "#{name}... "
unless File.exist?(repo_base_path)
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 2453079911d..d6d15285489 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -12,7 +12,7 @@ namespace :gitlab do
namespaces = Namespace.pluck(:path)
namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored
Gitlab.config.repositories.storages.each do |name, repository_storage|
- git_base_path = repository_storage['path']
+ git_base_path = repository_storage.legacy_disk_path
all_dirs = Dir.glob(git_base_path + '/*')
puts git_base_path.color(:yellow)
@@ -54,7 +54,7 @@ namespace :gitlab do
move_suffix = "+orphaned+#{Time.now.to_i}"
Gitlab.config.repositories.storages.each do |name, repository_storage|
- repo_root = repository_storage['path']
+ repo_root = repository_storage.legacy_disk_path
# Look for global repos (legacy, depth 1) and normal repos (depth 2)
IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
find.each_line do |path|
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 45e9a1a1c72..47ed522aec3 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -68,7 +68,7 @@ namespace :gitlab do
puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
puts "Repository storage paths:"
Gitlab.config.repositories.storages.each do |name, repository_storage|
- puts "- #{name}: \t#{repository_storage['path']}"
+ puts "- #{name}: \t#{repository_storage.legacy_disk_path}"
end
puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
puts "Git:\t\t#{Gitlab.config.git.bin_path}"
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
new file mode 100644
index 00000000000..a45e5ca91e0
--- /dev/null
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -0,0 +1,22 @@
+require 'logger'
+
+desc "GitLab | Migrate LFS objects to remote storage"
+namespace :gitlab do
+ namespace :lfs do
+ task migrate: :environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Starting transfer of LFS files to object storage')
+
+ LfsObject.with_files_stored_locally
+ .find_each(batch_size: 10) do |lfs_object|
+ begin
+ lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+
+ logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
+ rescue => e
+ logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+ end
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake
index 7728c485e8d..6b22499a5c8 100644
--- a/lib/tasks/gitlab/two_factor.rake
+++ b/lib/tasks/gitlab/two_factor.rake
@@ -1,7 +1,7 @@
namespace :gitlab do
namespace :two_factor do
desc "GitLab | Disable Two-factor authentication (2FA) for all users"
- task disable_for_all_users: :environment do
+ task disable_for_all_users: :gitlab_environment do
scope = User.with_two_factor
count = scope.count
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
new file mode 100644
index 00000000000..78e18992a8e
--- /dev/null
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -0,0 +1,34 @@
+namespace :gitlab do
+ namespace :uploads do
+ desc 'GitLab | Uploads | Migrate the uploaded files to object storage'
+ task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
+ batch_size = ENV.fetch('BATCH', 200).to_i
+ @to_store = ObjectStorage::Store::REMOTE
+ @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym
+ @uploader_class = args.uploader_class.constantize
+ @model_class = args.model_class.constantize
+
+ uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches
+ end
+
+ def enqueue_batch(batch, index)
+ job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
+ @model_class,
+ @mounted_as,
+ @to_store)
+ puts "Enqueued job ##{index}: #{job}"
+ rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
+ # continue for the next batch
+ puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
+ end
+
+ def uploads
+ Upload.class_eval { include EachBatch } unless Upload < EachBatch
+
+ Upload
+ .where(store: [nil, ObjectStorage::Store::LOCAL],
+ uploader: @uploader_class.to_s,
+ model_type: @model_class.base_class.sti_name)
+ end
+ end
+end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index 1c7a8a90f5c..af30ecb0e9b 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -7,8 +7,8 @@ task setup_postgresql: :environment do
require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes')
require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb')
- require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb')
require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb')
+ require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb')
NamespacesProjectsPathLowerIndexes.new.up
AddUsersLowerUsernameEmailIndexes.new.up
@@ -17,6 +17,6 @@ task setup_postgresql: :environment do
AddLowerPathIndexToRedirectRoutes.new.up
IndexRedirectRoutesPathForLike.new.up
AddIndexOnNamespacesLowerName.new.up
- ReworkRedirectRoutesIndexes.new.up
UsersNameLowerIndex.new.up
+ AddPathIndexToRedirectRoutes.new.up
end
diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake
index 3e01f91d32c..b52af81fc16 100644
--- a/lib/tasks/test.rake
+++ b/lib/tasks/test.rake
@@ -4,8 +4,3 @@ desc "GitLab | Run all tests"
task :test do
Rake::Task["gitlab:test"].invoke
end
-
-unless Rails.env.production?
- desc "GitLab | Run all tests on CI with simplecov"
- task test_ci: [:rubocop, :brakeman, :karma, :spinach, :spec]
-end
View Lorry Controller Status