diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/api.rb | 3 | ||||
| -rw-r--r-- | lib/api/entities.rb | 1 | ||||
| -rw-r--r-- | lib/api/helpers/internal_helpers.rb | 6 | ||||
| -rw-r--r-- | lib/api/internal.rb | 6 | ||||
| -rw-r--r-- | lib/api/runner.rb | 5 | ||||
| -rw-r--r-- | lib/api/settings.rb | 2 | ||||
| -rw-r--r-- | lib/backup/repository.rb | 56 | ||||
| -rw-r--r-- | lib/banzai/reference_parser/issue_parser.rb | 3 | ||||
| -rw-r--r-- | lib/gitlab/auth/user_access_denied_reason.rb | 8 | ||||
| -rw-r--r-- | lib/gitlab/background_migration/fill_file_store_job_artifact.rb | 20 | ||||
| -rw-r--r-- | lib/gitlab/background_migration/fill_file_store_lfs_object.rb | 20 | ||||
| -rw-r--r-- | lib/gitlab/background_migration/fill_store_upload.rb | 21 | ||||
| -rw-r--r-- | lib/gitlab/database/count.rb | 72 | ||||
| -rw-r--r-- | lib/gitlab/diff/file.rb | 7 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/storage_service.rb | 15 | ||||
| -rw-r--r-- | lib/gitlab/grape_logging/loggers/queue_duration_logger.rb | 26 | ||||
| -rw-r--r-- | lib/google_api/cloud_platform/client.rb | 17 |
17 files changed, 219 insertions, 69 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index de20b2b8e67..206fabe5c43 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -15,7 +15,8 @@ module API include: [ GrapeLogging::Loggers::FilterParameters.new, GrapeLogging::Loggers::ClientEnv.new, - Gitlab::GrapeLogging::Loggers::UserLogger.new + Gitlab::GrapeLogging::Loggers::UserLogger.new, + Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new ] allow_access_with_scope :api diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 174c5af91d5..3e615f7ac05 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1020,6 +1020,7 @@ module API class Job < JobBasic expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :runner, with: Runner + expose :artifacts_expire_at end class JobBasicWithProject < JobBasic diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index abe3d353984..83151be82ad 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -89,12 +89,6 @@ module API end end - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - def repository_path - repository.path_to_repo - end - # Return the Gitaly Address if it is enabled def gitaly_payload(action) return unless %w[git-receive-pack git-upload-pack git-upload-archive].include?(action) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index a3dac36b8b6..a9803be9f69 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -59,7 +59,11 @@ module API status: true, gl_repository: gl_repository, gl_username: user&.username, - repository_path: repository_path, + + # This repository_path is a bogus value but gitlab-shell still requires + # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135 + repository_path: '/', + gitaly: gitaly_payload(params[:action]) } end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index a7f1cb1131f..5b7ae89440c 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -123,6 +123,7 @@ module API end put '/:id' do job = authenticate_job! + forbidden!('Job is not running') unless job.running? job.trace.set(params[:trace]) if params[:trace] @@ -131,9 +132,9 @@ module API case params[:state].to_s when 'success' - job.success + job.success! when 'failed' - job.drop(params[:failure_reason] || :unknown_failure) + job.drop!(params[:failure_reason] || :unknown_failure) end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index e31c332b6e4..d727ad59367 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -24,7 +24,7 @@ module API optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' - optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index c3360c391af..84670d6582e 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -73,29 +73,40 @@ module Backup end def prepare_directories - # TODO: Need to find a way to do this for gitaly - # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194 - Gitlab.config.repositories.storages.each do |name, repository_storage| - path = repository_storage.legacy_disk_path - next unless File.exist?(path) - - # Move all files in the existing repos directory except . and .. to - # repositories.old.<timestamp> directory - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] - - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) + delete_all_repositories(name, repository_storage) + end + end + + def delete_all_repositories(name, repository_storage) + gitaly_migrate(:delete_all_repositories) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories + else + local_delete_all_repositories(name, repository_storage) end end end + def local_delete_all_repositories(name, repository_storage) + path = repository_storage.legacy_disk_path + return unless File.exist?(path) + + # Move all files in the existing repos directory except . and .. to + # repositories.old.<timestamp> directory + bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) + FileUtils.mkdir_p(bk_repos_path, mode: 0700) + files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + + begin + FileUtils.mv(files, bk_repos_path) + rescue Errno::EACCES + access_denied_error(path) + rescue Errno::EBUSY + resource_busy_error(path) + end + end + def restore_custom_hooks(project) # TODO: Need to find a way to do this for gitaly # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 @@ -113,6 +124,7 @@ module Backup def restore prepare_directories gitlab_shell = Gitlab::Shell.new + Project.find_each(batch_size: 1000) do |project| progress.print " * #{project.full_path} ... " path_to_project_bundle = path_to_bundle(project) @@ -121,7 +133,6 @@ module Backup restore_repo_success = nil if File.exist?(path_to_project_bundle) begin - gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists? project.repository.create_from_bundle path_to_project_bundle restore_repo_success = true rescue => e @@ -146,7 +157,6 @@ module Backup if File.exist?(path_to_wiki_bundle) progress.print " * #{wiki.full_path} ... " begin - gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists? wiki.repository.create_from_bundle(path_to_wiki_bundle) progress.puts "[DONE]".color(:green) rescue => e @@ -224,5 +234,11 @@ module Backup def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end + + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) + Gitlab::GitalyClient.migrate(method, status: status, &block) + rescue GRPC::NotFound, GRPC::BadStatus => e + raise Error, e + end end end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 6bee5ea15b9..7b5915899cf 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -69,7 +69,8 @@ module Banzai { group: [:owners, :group_members] }, :invited_groups, :project_members, - :project_feature + :project_feature, + :route ] } ), diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index af310aa12fc..1893cb001b2 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -8,12 +8,12 @@ module Gitlab def rejection_message case rejection_type when :internal - 'This action cannot be performed by internal users' + "This action cannot be performed by internal users" when :terms_not_accepted - 'You must accept the Terms of Service in order to perform this action. '\ - 'Please access GitLab from a web browser to accept these terms.' + "You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\ + "Please access GitLab from a web browser to accept these terms." else - 'Your account has been blocked.' + "Your account has been blocked." end end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb new file mode 100644 index 00000000000..22b0ac71920 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreJobArtifact + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + end + + def perform(start_id, stop_id) + FillFileStoreJobArtifact::JobArtifact + .where(file_store: nil) + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb new file mode 100644 index 00000000000..d0816ae3ed5 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreLfsObject + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + def perform(start_id, stop_id) + FillFileStoreLfsObject::LfsObject + .where(file_store: nil) + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb new file mode 100644 index 00000000000..94c65459a67 --- /dev/null +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillStoreUpload + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillStoreUpload::Upload + .where(store: nil) + .where(id: (start_id..stop_id)) + .update_all(store: 1) + end + end + end +end diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index 3374203960e..5f549ed2b3c 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -17,31 +17,69 @@ module Gitlab ].freeze end - def self.approximate_count(model) - return model.count unless Gitlab::Database.postgresql? + # Takes in an array of models and returns a Hash for the approximate + # counts for them. If the model's table has not been vacuumed or + # analyzed recently, simply run the Model.count to get the data. + # + # @param [Array] + # @return [Hash] of Model -> count mapping + def self.approximate_counts(models) + table_to_model_map = models.each_with_object({}) do |model, hash| + hash[model.table_name] = model + end - execute_estimate_if_updated_recently(model) || model.count - end + table_names = table_to_model_map.keys + counts_by_table_name = Gitlab::Database.postgresql? ? reltuples_from_recently_updated(table_names) : {} - def self.execute_estimate_if_updated_recently(model) - ActiveRecord::Base.connection.select_value(postgresql_estimate_query(model)).to_i if reltuples_updated_recently?(model) - rescue *CONNECTION_ERRORS + # Convert table -> count to Model -> count + counts_by_model = counts_by_table_name.each_with_object({}) do |pair, hash| + model = table_to_model_map[pair.first] + hash[model] = pair.second + end + + missing_tables = table_names - counts_by_table_name.keys + + missing_tables.each do |table| + model = table_to_model_map[table] + counts_by_model[model] = model.count + end + + counts_by_model end - def self.reltuples_updated_recently?(model) - time = "to_timestamp(#{1.hour.ago.to_i})" - query = <<~SQL - SELECT 1 FROM pg_stat_user_tables WHERE relname = '#{model.table_name}' AND - (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) - SQL + # Returns a hash of the table names that have recently updated tuples. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def self.reltuples_from_recently_updated(table_names) + query = postgresql_estimate_query(table_names) + rows = [] - ActiveRecord::Base.connection.select_all(query).count > 0 + # Querying tuple stats only works on the primary. Due to load + # balancing, we need to ensure this query hits the load balancer. The + # easiest way to do this is to start a transaction. + ActiveRecord::Base.transaction do + rows = ActiveRecord::Base.connection.select_all(query) + end + + rows.each_with_object({}) { |row, data| data[row['table_name']] = row['estimate'].to_i } rescue *CONNECTION_ERRORS - false + {} end - def self.postgresql_estimate_query(model) - "SELECT reltuples::bigint AS estimate FROM pg_class where relname = '#{model.table_name}'" + # Generates the PostgreSQL query to return the tuples for tables + # that have been vacuumed or analyzed in the last hour. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def self.postgresql_estimate_query(table_names) + time = "to_timestamp(#{1.hour.ago.to_i})" + <<~SQL + SELECT pg_class.relname AS table_name, reltuples::bigint AS estimate FROM pg_class + LEFT JOIN pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname + WHERE pg_class.relname IN (#{table_names.map { |table| "'#{table}'" }.join(',')}) + AND (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) + SQL end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 014854da55c..765fb0289a8 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -76,6 +76,13 @@ module Gitlab line_code(line) if line end + # Returns the raw diff content up to the given line index + def diff_hunk(diff_line) + # Adding 2 because of the @@ diff header and Enum#take should consider + # an extra line, because we're passing an index. + raw_diff.each_line.take(diff_line.index + 2).join + end + def old_sha diff_refs&.base_sha end diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb new file mode 100644 index 00000000000..eb0e910665b --- /dev/null +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -0,0 +1,15 @@ +module Gitlab + module GitalyClient + class StorageService + def initialize(storage) + @storage = storage + end + + # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. + def delete_all_repositories + request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) + GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request) + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb new file mode 100644 index 00000000000..0adac79f25a --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb @@ -0,0 +1,26 @@ +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log how much time an API request was queued by Workhorse. +module Gitlab + module GrapeLogging + module Loggers + class QueueDurationLogger < ::GrapeLogging::Loggers::Base + attr_accessor :start_time + + def before + @start_time = Time.now + end + + def parameters(request, _) + proxy_start = request.env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence + + return {} unless proxy_start && start_time + + # Time in milliseconds since gitlab-workhorse started the request + duration = (start_time.to_f * 1_000 - proxy_start.to_f / 1_000_000).round(2) + + { 'queue_duration': duration } + end + end + end + end +end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index f30dd995695..36859b4d025 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,3 +1,4 @@ +require 'google/apis/compute_v1' require 'google/apis/container_v1' require 'google/apis/cloudbilling_v1' require 'google/apis/cloudresourcemanager_v1' @@ -42,22 +43,6 @@ module GoogleApi true end - def projects_list - service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new - service.authorization = access_token - - service.fetch_all(items: :projects) do |token| - service.list_projects(page_token: token, options: user_agent_header) - end - end - - def projects_get_billing_info(project_id) - service = Google::Apis::CloudbillingV1::CloudbillingService.new - service.authorization = access_token - - service.get_project_billing_info("projects/#{project_id}", options: user_agent_header) - end - def projects_zones_clusters_get(project_id, zone, cluster_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token |