summaryrefslogtreecommitdiff
path: root/lib/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'lib/tasks')
-rw-r--r--lib/tasks/frontend.rake21
-rw-r--r--lib/tasks/gitlab/cleanup.rake52
-rw-r--r--lib/tasks/gitlab/db.rake23
-rw-r--r--lib/tasks/gitlab/features.rake14
-rw-r--r--lib/tasks/gitlab/setup.rake3
-rw-r--r--lib/tasks/gitlab/update_templates.rake62
-rw-r--r--lib/tasks/karma.rake11
-rw-r--r--lib/tasks/migrate/add_limits_mysql.rake17
-rw-r--r--lib/tasks/spec.rake63
9 files changed, 146 insertions, 120 deletions
diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake
new file mode 100644
index 00000000000..1cac7520227
--- /dev/null
+++ b/lib/tasks/frontend.rake
@@ -0,0 +1,21 @@
+unless Rails.env.production?
+ namespace :frontend do
+ desc 'GitLab | Frontend | Generate fixtures for JavaScript tests'
+ RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args|
+ args.with_defaults(pattern: '{spec,ee/spec}/frontend/fixtures/*.rb')
+ ENV['NO_KNAPSACK'] = 'true'
+ t.pattern = args[:pattern]
+ t.rspec_opts = '--format documentation'
+ end
+
+ desc 'GitLab | Frontend | Run JavaScript tests'
+ task tests: ['yarn:check'] do
+ sh "yarn test" do |ok, res|
+ abort('rake frontend:tests failed') unless ok
+ end
+ end
+ end
+
+ desc 'GitLab | Frontend | Shortcut for frontend:fixtures and frontend:tests'
+ task frontend: ['frontend:fixtures', 'frontend:tests']
+end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 88172e26c67..4d854cd178d 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -127,6 +127,58 @@ namespace :gitlab do
end
end
+ namespace :sessions do
+ desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys"
+ task active_sessions_lookup_keys: :gitlab_environment do
+ session_key_pattern = "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:*"
+ last_save_check = Time.at(0)
+ wait_time = 10.seconds
+ cursor = 0
+ total_users_scanned = 0
+
+ Gitlab::Redis::SharedState.with do |redis|
+ begin
+ cursor, keys = redis.scan(cursor, match: session_key_pattern)
+ total_users_scanned += keys.count
+
+ if last_save_check < Time.now - 1.second
+ while redis.info('persistence')['rdb_bgsave_in_progress'] == '1'
+ puts "BGSAVE in progress, waiting #{wait_time} seconds"
+ sleep(wait_time)
+ end
+ last_save_check = Time.now
+ end
+
+ keys.each do |key|
+ user_id = key.split(':').last
+
+ lookup_key_count = redis.scard(key)
+
+ session_ids = ActiveSession.session_ids_for_user(user_id)
+ entries = ActiveSession.raw_active_session_entries(session_ids, user_id)
+ session_ids_and_entries = session_ids.zip(entries)
+
+ inactive_session_ids = session_ids_and_entries.map do |session_id, session|
+ session_id if session.nil?
+ end.compact
+
+ redis.pipelined do |conn|
+ inactive_session_ids.each do |session_id|
+ conn.srem(key, session_id)
+ end
+ end
+
+ if inactive_session_ids
+ puts "deleted #{inactive_session_ids.count} out of #{lookup_key_count} lookup keys for User ##{user_id}"
+ end
+ end
+ end while cursor.to_i != 0
+
+ puts "--- All done! Total number of scanned users: #{total_users_scanned}"
+ end
+ end
+ end
+
def remove?
ENV['REMOVE'] == 'true'
end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 4e7a8adbef6..1961f64659c 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -26,26 +26,19 @@ namespace :gitlab do
task drop_tables: :environment do
connection = ActiveRecord::Base.connection
- # If MySQL, turn off foreign key checks
- connection.execute('SET FOREIGN_KEY_CHECKS=0') if Gitlab::Database.mysql?
-
- # connection.tables is deprecated in MySQLAdapter, but in PostgreSQLAdapter
- # data_sources returns both views and tables, so use #tables instead
- tables = Gitlab::Database.mysql? ? connection.data_sources : connection.tables
+ # In PostgreSQLAdapter, data_sources returns both views and tables, so use
+ # #tables instead
+ tables = connection.tables
# Removes the entry from the array
tables.delete 'schema_migrations'
# Truncate schema_migrations to ensure migrations re-run
- connection.execute('TRUNCATE schema_migrations') if connection.data_source_exists? 'schema_migrations'
+ connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'
# Drop tables with cascade to avoid dependent table errors
# PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
- # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html
# Add `IF EXISTS` because cascade could have already deleted a table.
tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
-
- # If MySQL, re-enable foreign key checks
- connection.execute('SET FOREIGN_KEY_CHECKS=1') if Gitlab::Database.mysql?
end
desc 'Configures the database by running migrate, or by loading the schema and seeding if needed'
@@ -77,5 +70,13 @@ namespace :gitlab do
Gitlab::DowntimeCheck.new.check_and_print(migrations)
end
+
+ desc 'Sets up EE specific database functionality'
+
+ if Gitlab.ee?
+ task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate]
+ else
+ task :setup_ee
+ end
end
end
diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake
index d88bcca0819..9cf568c07fe 100644
--- a/lib/tasks/gitlab/features.rake
+++ b/lib/tasks/gitlab/features.rake
@@ -10,14 +10,22 @@ namespace :gitlab do
set_rugged_feature_flags(false)
puts 'All Rugged feature flags were disabled.'
end
+
+ task unset_rugged: :environment do
+ set_rugged_feature_flags(nil)
+ puts 'All Rugged feature flags were unset.'
+ end
end
def set_rugged_feature_flags(status)
Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag|
- if status
- Feature.enable(flag)
- else
+ case status
+ when nil
Feature.get(flag).remove
+ when true
+ Feature.enable(flag)
+ when false
+ Feature.disable(flag)
end
end
end
diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake
index e876b23d43f..5d86d6e466c 100644
--- a/lib/tasks/gitlab/setup.rake
+++ b/lib/tasks/gitlab/setup.rake
@@ -31,7 +31,6 @@ namespace :gitlab do
terminate_all_connections unless Rails.env.production?
Rake::Task["db:reset"].invoke
- Rake::Task["add_limits_mysql"].invoke
Rake::Task["setup_postgresql"].invoke
Rake::Task["db:seed_fu"].invoke
rescue Gitlab::TaskAbortedByUserError
@@ -46,8 +45,6 @@ namespace :gitlab do
# method terminates all the connections so that a subsequent DROP
# will work.
def self.terminate_all_connections
- return false unless Gitlab::Database.postgresql?
-
cmd = <<~SQL
SELECT pg_terminate_backend(pg_stat_activity.pid)
FROM pg_stat_activity
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index e058e9fe069..8267c235a7f 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -5,25 +5,43 @@ namespace :gitlab do
end
desc "GitLab | Update project templates"
- task :update_project_templates do
- include Gitlab::ImportExport::CommandLineUtil
+ task :update_project_templates, [] => :environment do |_task, args|
+ # we need an instance method from Gitlab::ImportExport::CommandLineUtil and don't
+ # want to include it in the task, as this would affect subsequent tasks as well
+ downloader = Class.new do
+ extend Gitlab::ImportExport::CommandLineUtil
+
+ def self.call(uploader, upload_path)
+ download_or_copy_upload(uploader, upload_path)
+ end
+ end
+
+ template_names = args.extras.to_set
if Rails.env.production?
- puts "This rake task is not meant fo production instances".red
- exit(1)
+ raise "This rake task is not meant for production instances"
end
admin = User.find_by(admin: true)
unless admin
- puts "No admin user could be found".red
- exit(1)
+ raise "No admin user could be found"
end
- Gitlab::ProjectTemplate.all.each do |template|
+ tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}"
+ puts "Creating temporary namespace #{tmp_namespace_path}"
+ tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path)
+
+ templates = if template_names.empty?
+ Gitlab::ProjectTemplate.all
+ else
+ Gitlab::ProjectTemplate.all.select { |template| template_names.include?(template.name) }
+ end
+
+ templates.each do |template|
params = {
import_url: template.clone_url,
- namespace_id: admin.namespace.id,
+ namespace_id: tmp_namespace.id,
path: template.name,
skip_wiki: true
}
@@ -32,19 +50,17 @@ namespace :gitlab do
project = Projects::CreateService.new(admin, params).execute
unless project.persisted?
- puts project.errors.messages
- exit(1)
+ raise "Failed to create project: #{project.errors.messages}"
end
loop do
- if project.finished?
+ if project.import_finished?
puts "Import finished for #{template.name}"
break
end
- if project.failed?
- puts "Failed to import from #{project_params[:import_url]}".red
- exit(1)
+ if project.import_failed?
+ raise "Failed to import from #{project_params[:import_url]}"
end
puts "Waiting for the import to finish"
@@ -54,11 +70,23 @@ namespace :gitlab do
end
Projects::ImportExport::ExportService.new(project, admin).execute
- download_or_copy_upload(project.export_file, template.archive_path)
- Projects::DestroyService.new(admin, project).execute
+ downloader.call(project.export_file, template.archive_path)
+
+ unless Projects::DestroyService.new(project, admin).execute
+ puts "Failed to destroy project #{template.name} (but namespace will be cleaned up later)"
+ end
+
puts "Exported #{template.name}".green
end
- puts "Done".green
+
+ success = true
+ ensure
+ if tmp_namespace
+ puts "Destroying temporary namespace #{tmp_namespace_path}"
+ tmp_namespace.destroy
+ end
+
+ puts "Done".green if success
end
def update(template)
diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake
index 2dc14183fa3..36590010406 100644
--- a/lib/tasks/karma.rake
+++ b/lib/tasks/karma.rake
@@ -1,15 +1,8 @@
unless Rails.env.production?
namespace :karma do
+ # alias exists for legacy reasons
desc 'GitLab | Karma | Generate fixtures for JavaScript tests'
- task fixtures: ['karma:rspec_fixtures']
-
- desc 'GitLab | Karma | Generate fixtures using RSpec'
- RSpec::Core::RakeTask.new(:rspec_fixtures, [:pattern]) do |t, args|
- args.with_defaults(pattern: '{spec,ee/spec}/javascripts/fixtures/*.rb')
- ENV['NO_KNAPSACK'] = 'true'
- t.pattern = args[:pattern]
- t.rspec_opts = '--format documentation'
- end
+ task fixtures: ['frontend:fixtures']
desc 'GitLab | Karma | Run JavaScript tests'
task tests: ['yarn:check'] do
diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake
deleted file mode 100644
index c77fa49d586..00000000000
--- a/lib/tasks/migrate/add_limits_mysql.rake
+++ /dev/null
@@ -1,17 +0,0 @@
-require Rails.root.join('db/migrate/limits_to_mysql')
-require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
-require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
-require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
-require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql')
-require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql')
-
-desc "GitLab | Add limits to strings in mysql database"
-task add_limits_mysql: :environment do
- puts "Adding limits to schema.rb for mysql"
- LimitsToMysql.new.up
- MarkdownCacheLimitsToMysql.new.up
- MergeRequestDiffFileLimitsToMysql.new.up
- LimitsCiBuildTraceChunksRawDataForMysql.new.up
- IncreaseMysqlTextLimitForGpgKeys.new.up
- PrometheusMetricsLimitsToMysql.new.up
-end
diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake
index c881ad4cf12..bf18332a8eb 100644
--- a/lib/tasks/spec.rake
+++ b/lib/tasks/spec.rake
@@ -2,8 +2,6 @@
return if Rails.env.production?
-Rake::Task["spec"].clear if Rake::Task.task_defined?('spec')
-
namespace :spec do
desc 'GitLab | RSpec | Run unit tests'
RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args|
@@ -26,63 +24,8 @@ namespace :spec do
t.rspec_opts = args[:rspec_opts]
end
- desc '[Deprecated] Use the "bin/rspec --tag api" instead'
- task :api do
- cmds = [
- %w(rake gitlab:setup),
- %w(rspec spec --tag @api)
- ]
- run_commands(cmds)
- end
-
- desc '[Deprecated] Use the "spec:system" task instead'
- task :feature do
- cmds = [
- %w(rake gitlab:setup),
- %w(rspec spec --tag @feature)
- ]
- run_commands(cmds)
- end
-
- desc '[Deprecated] Use "bin/rspec spec/models" instead'
- task :models do
- cmds = [
- %w(rake gitlab:setup),
- %w(rspec spec --tag @models)
- ]
- run_commands(cmds)
- end
-
- desc '[Deprecated] Use "bin/rspec spec/services" instead'
- task :services do
- cmds = [
- %w(rake gitlab:setup),
- %w(rspec spec --tag @services)
- ]
- run_commands(cmds)
- end
-
- desc '[Deprecated] Use "bin/rspec spec/lib" instead'
- task :lib do
- cmds = [
- %w(rake gitlab:setup),
- %w(rspec spec --tag @lib)
- ]
- run_commands(cmds)
- end
-end
-
-desc "GitLab | Run specs"
-task :spec do
- cmds = [
- %w(rake gitlab:setup),
- %w(rspec spec)
- ]
- run_commands(cmds)
-end
-
-def run_commands(cmds)
- cmds.each do |cmd|
- system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!")
+ desc 'Run the code examples in spec/requests/api'
+ RSpec::Core::RakeTask.new(:api) do |t|
+ t.pattern = 'spec/requests/api/**/*_spec.rb'
end
end
View Lorry Controller Status