From 45b3bcc39631729ce573210ef6201c2cc59d7cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 16 Mar 2018 17:50:48 +0100 Subject: Add zones_list and machine_types_list to Google Cloud API client --- lib/google_api/cloud_platform/client.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'lib') diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index f30dd995695..f7a00974efc 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,3 +1,4 @@ +require 'google/apis/compute_v1' require 'google/apis/container_v1' require 'google/apis/cloudbilling_v1' require 'google/apis/cloudresourcemanager_v1' @@ -58,6 +59,24 @@ module GoogleApi service.get_project_billing_info("projects/#{project_id}", options: user_agent_header) end + def projects_zones_list(project_id) + service = Google::Apis::ComputeV1::ComputeService.new + service.authorization = access_token + + service.fetch_all do |token| + service.list_zones(project_id, page_token: token, options: user_agent_header) + end + end + + def projects_zones_machine_types_list(project_id, zone) + service = Google::Apis::ComputeV1::ComputeService.new + service.authorization = access_token + + service.fetch_all do |token| + service.list_machine_types(project_id, zone, page_token: token, options: user_agent_header) + end + end + def projects_zones_clusters_get(project_id, zone, cluster_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token -- cgit v1.2.1 From b22959cc040384ab942bd41a69cdb4f2ffea0f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 11 May 2018 18:45:45 +0200 Subject: Remove redundant GoogleApi::CloudPlatform::Client methods The methods removed are #projects_list #get_project_billing_info as well as #project_zones_list and #project_zone_machine_types_list --- lib/google_api/cloud_platform/client.rb | 34 --------------------------------- 1 file changed, 34 deletions(-) (limited to 'lib') diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index f7a00974efc..36859b4d025 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -43,40 +43,6 @@ module GoogleApi true end - def projects_list - service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new - service.authorization = access_token - - service.fetch_all(items: :projects) do |token| - service.list_projects(page_token: token, options: user_agent_header) - end - end - - def projects_get_billing_info(project_id) - service = Google::Apis::CloudbillingV1::CloudbillingService.new - service.authorization = access_token - - service.get_project_billing_info("projects/#{project_id}", options: user_agent_header) - end - - def projects_zones_list(project_id) - service = Google::Apis::ComputeV1::ComputeService.new - service.authorization = access_token - - service.fetch_all do |token| - service.list_zones(project_id, page_token: token, options: user_agent_header) - end - end - - def projects_zones_machine_types_list(project_id, zone) - service = Google::Apis::ComputeV1::ComputeService.new - service.authorization = access_token - - service.fetch_all do |token| - service.list_machine_types(project_id, zone, page_token: token, options: user_agent_header) - end - end - def projects_zones_clusters_get(project_id, zone, cluster_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token -- cgit v1.2.1 From a33d591602b40d224b4fdb74300e3b0d96bda7df Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 17 May 2018 15:19:32 +0200 Subject: Don't pass the repository path to Gitlab Shell This isn't a requirement, and removes deprecated method calls --- lib/api/helpers/internal_helpers.rb | 6 ------ lib/api/internal.rb | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index abe3d353984..83151be82ad 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -89,12 +89,6 @@ module API end end - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - def repository_path - repository.path_to_repo - end - # Return the Gitaly Address if it is enabled def gitaly_payload(action) return unless %w[git-receive-pack git-upload-pack git-upload-archive].include?(action) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 6b72caea8fd..798a8dbb929 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -59,7 +59,7 @@ module API status: true, gl_repository: gl_repository, gl_username: user&.username, - repository_path: repository_path, + repository_path: nil, gitaly: gitaly_payload(params[:action]) } end -- cgit v1.2.1 From 7a139c1602016485b8a2038dfc6121e47039c669 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 24 May 2018 09:46:30 +0200 Subject: Add username to terms message in git and API calls This will make it clearer to users which account is being used to make the API/git call. So they know which account needs to be used to accept the terms. Closes #46649 --- lib/gitlab/auth/user_access_denied_reason.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index af310aa12fc..1893cb001b2 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -8,12 +8,12 @@ module Gitlab def rejection_message case rejection_type when :internal - 'This action cannot be performed by internal users' + "This action cannot be performed by internal users" when :terms_not_accepted - 'You must accept the Terms of Service in order to perform this action. '\ - 'Please access GitLab from a web browser to accept these terms.' + "You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\ + "Please access GitLab from a web browser to accept these terms." else - 'Your account has been blocked.' + "Your account has been blocked." end end -- cgit v1.2.1 From bb8f2520b4254c9dabe377df48e29c5f17894a1d Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Wed, 16 May 2018 12:46:18 -0300 Subject: Persist truncated note diffs on a new table We request Gitaly in a N+1 manner to build discussion diffs. Once the diffs are from different revisions, it's hard to make a single request to the service in order to build the whole response. With this change we solve this problem and simplify a lot fetching this piece of info. --- lib/gitlab/diff/file.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 014854da55c..765fb0289a8 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -76,6 +76,13 @@ module Gitlab line_code(line) if line end + # Returns the raw diff content up to the given line index + def diff_hunk(diff_line) + # Adding 2 because of the @@ diff header and Enum#take should consider + # an extra line, because we're passing an index. + raw_diff.each_line.take(diff_line.index + 2).join + end + def old_sha diff_refs&.base_sha end -- cgit v1.2.1 From 8fad07383ada021fc995294fd0fe0f77fe37da35 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 25 May 2018 13:45:29 +0200 Subject: Return '/' as a bogus repo path to gitlab-shell --- lib/api/internal.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 798a8dbb929..a41270fa85f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -59,7 +59,11 @@ module API status: true, gl_repository: gl_repository, gl_username: user&.username, - repository_path: nil, + + # This repository_path is a bogus value but gitlab-shell still requires + # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135 + repository_path: '/', + gitaly: gitaly_payload(params[:action]) } end -- cgit v1.2.1 From b6125f7045d2bed4aadf798dde4c547e2047e039 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 25 May 2018 14:28:16 -0700 Subject: Fix fast admin counters not working when PostgreSQL has secondaries This commit does a number of things: 1. Reduces the number of queries needed by perform a single query to get all the tuples for the relevant rows. 2. Uses a transaction to query the tuple counts to ensure that the data is retrieved from the primary. Closes #46742 --- lib/gitlab/database/count.rb | 72 +++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index 3374203960e..5f549ed2b3c 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -17,31 +17,69 @@ module Gitlab ].freeze end - def self.approximate_count(model) - return model.count unless Gitlab::Database.postgresql? + # Takes in an array of models and returns a Hash for the approximate + # counts for them. If the model's table has not been vacuumed or + # analyzed recently, simply run the Model.count to get the data. + # + # @param [Array] + # @return [Hash] of Model -> count mapping + def self.approximate_counts(models) + table_to_model_map = models.each_with_object({}) do |model, hash| + hash[model.table_name] = model + end - execute_estimate_if_updated_recently(model) || model.count - end + table_names = table_to_model_map.keys + counts_by_table_name = Gitlab::Database.postgresql? ? reltuples_from_recently_updated(table_names) : {} - def self.execute_estimate_if_updated_recently(model) - ActiveRecord::Base.connection.select_value(postgresql_estimate_query(model)).to_i if reltuples_updated_recently?(model) - rescue *CONNECTION_ERRORS + # Convert table -> count to Model -> count + counts_by_model = counts_by_table_name.each_with_object({}) do |pair, hash| + model = table_to_model_map[pair.first] + hash[model] = pair.second + end + + missing_tables = table_names - counts_by_table_name.keys + + missing_tables.each do |table| + model = table_to_model_map[table] + counts_by_model[model] = model.count + end + + counts_by_model end - def self.reltuples_updated_recently?(model) - time = "to_timestamp(#{1.hour.ago.to_i})" - query = <<~SQL - SELECT 1 FROM pg_stat_user_tables WHERE relname = '#{model.table_name}' AND - (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) - SQL + # Returns a hash of the table names that have recently updated tuples. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def self.reltuples_from_recently_updated(table_names) + query = postgresql_estimate_query(table_names) + rows = [] - ActiveRecord::Base.connection.select_all(query).count > 0 + # Querying tuple stats only works on the primary. Due to load + # balancing, we need to ensure this query hits the load balancer. The + # easiest way to do this is to start a transaction. + ActiveRecord::Base.transaction do + rows = ActiveRecord::Base.connection.select_all(query) + end + + rows.each_with_object({}) { |row, data| data[row['table_name']] = row['estimate'].to_i } rescue *CONNECTION_ERRORS - false + {} end - def self.postgresql_estimate_query(model) - "SELECT reltuples::bigint AS estimate FROM pg_class where relname = '#{model.table_name}'" + # Generates the PostgreSQL query to return the tuples for tables + # that have been vacuumed or analyzed in the last hour. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def self.postgresql_estimate_query(table_names) + time = "to_timestamp(#{1.hour.ago.to_i})" + <<~SQL + SELECT pg_class.relname AS table_name, reltuples::bigint AS estimate FROM pg_class + LEFT JOIN pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname + WHERE pg_class.relname IN (#{table_names.map { |table| "'#{table}'" }.join(',')}) + AND (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) + SQL end end end -- cgit v1.2.1 From 4131a49bb72ab62c9384050d90ec268b4f3d76aa Mon Sep 17 00:00:00 2001 From: Semyon Pupkov Date: Thu, 10 May 2018 12:58:19 +0500 Subject: Expose artifacts_expire_at field for job entity in api https://gitlab.com/gitlab-org/gitlab-ce/issues/45798 --- lib/api/entities.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 174c5af91d5..3e615f7ac05 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1020,6 +1020,7 @@ module API class Job < JobBasic expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :runner, with: Runner + expose :artifacts_expire_at end class JobBasicWithProject < JobBasic -- cgit v1.2.1 From ffacc06efb0f71d8aed18c6517fc18d168873cf2 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 23 May 2018 16:42:24 +0900 Subject: Ignore writing traces if it's already been archived --- lib/gitlab/ci/trace.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index fe15fabc2e8..58a0ec6dea4 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -77,7 +77,9 @@ module Gitlab def write(mode) stream = Gitlab::Ci::Trace::Stream.new do - if current_path + if trace_artifact + # no op. It's already archived. + elsif current_path File.open(current_path, mode) elsif Feature.enabled?('ci_enable_live_trace') Gitlab::Ci::Trace::ChunkedIO.new(job) -- cgit v1.2.1 From 88b93f55a13f51776991d0437d8a63e18e777419 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 27 May 2018 21:08:24 +0900 Subject: Revert trace change --- lib/gitlab/ci/trace.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 58a0ec6dea4..30e3b523470 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -59,9 +59,7 @@ module Gitlab def read stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact - trace_artifact.open - elsif job.trace_chunks.any? + if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) elsif current_path File.open(current_path, "rb") -- cgit v1.2.1 From db55fa594274c95ffade4089aad3d463f918d4c3 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 27 May 2018 21:09:58 +0900 Subject: Disallow updating job stauts if it's not running --- lib/api/runner.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index a7f1cb1131f..59898cd2477 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -123,6 +123,7 @@ module API end put '/:id' do job = authenticate_job! + forbidden!('Job is not running') unless job.running? job.trace.set(params[:trace]) if params[:trace] -- cgit v1.2.1 From a98a462fb21867f3d62b8a60871d4068952b9796 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 27 May 2018 21:23:37 +0900 Subject: Revert correctly --- lib/api/runner.rb | 4 ++-- lib/gitlab/ci/trace.rb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 59898cd2477..5b7ae89440c 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -132,9 +132,9 @@ module API case params[:state].to_s when 'success' - job.success + job.success! when 'failed' - job.drop(params[:failure_reason] || :unknown_failure) + job.drop!(params[:failure_reason] || :unknown_failure) end end diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 30e3b523470..fe15fabc2e8 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -59,7 +59,9 @@ module Gitlab def read stream = Gitlab::Ci::Trace::Stream.new do - if job.trace_chunks.any? + if trace_artifact + trace_artifact.open + elsif job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) elsif current_path File.open(current_path, "rb") @@ -75,9 +77,7 @@ module Gitlab def write(mode) stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact - # no op. It's already archived. - elsif current_path + if current_path File.open(current_path, mode) elsif Feature.enabled?('ci_enable_live_trace') Gitlab::Ci::Trace::ChunkedIO.new(job) -- cgit v1.2.1 From 76f0d7fe6e1a80d5b07eaa792f33d62ec8736d0d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 24 Apr 2018 16:43:19 +0900 Subject: Add background migration to fill file stores --- .../fill_file_store_build_archive.rb | 21 +++++++++++++++++++++ .../fill_file_store_build_metadata.rb | 21 +++++++++++++++++++++ .../fill_file_store_job_artifact.rb | 20 ++++++++++++++++++++ .../fill_file_store_lfs_object.rb | 20 ++++++++++++++++++++ .../background_migration/fill_file_store_upload.rb | 21 +++++++++++++++++++++ 5 files changed, 103 insertions(+) create mode 100644 lib/gitlab/background_migration/fill_file_store_build_archive.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_build_metadata.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_job_artifact.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_lfs_object.rb create mode 100644 lib/gitlab/background_migration/fill_file_store_upload.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/fill_file_store_build_archive.rb b/lib/gitlab/background_migration/fill_file_store_build_archive.rb new file mode 100644 index 00000000000..02d744b7a74 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_build_archive.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreBuildArchive + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillFileStoreBuildArchive::Build + .where('artifacts_file_store = NULL') + .where(id: (start_id..stop_id)) + .update_all(artifacts_file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_build_metadata.rb b/lib/gitlab/background_migration/fill_file_store_build_metadata.rb new file mode 100644 index 00000000000..bf1a10b104e --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_build_metadata.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreBuildMetadata + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillFileStoreBuildMetadata::Build + .where('artifacts_metadata_store = NULL') + .where(id: (start_id..stop_id)) + .update_all(artifacts_metadata_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb new file mode 100644 index 00000000000..dfd1e7191d2 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreJobArtifact + class JobArtifact < ActiveRecord::Base + self.table_name = 'ci_job_artifacts' + end + + def perform(start_id, stop_id) + FillFileStoreJobArtifact::JobArtifact + .where('file_store = NULL') + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb new file mode 100644 index 00000000000..d423bcfba95 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreLfsObject + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + def perform(start_id, stop_id) + FillFileStoreLfsObject::LfsObject + .where('file_store = NULL') + .where(id: (start_id..stop_id)) + .update_all(file_store: 1) + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_upload.rb b/lib/gitlab/background_migration/fill_file_store_upload.rb new file mode 100644 index 00000000000..14f4c667ed8 --- /dev/null +++ b/lib/gitlab/background_migration/fill_file_store_upload.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillFileStoreUpload + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + FillFileStoreUpload::Upload + .where('store = NULL') + .where(id: (start_id..stop_id)) + .update_all(store: 1) + end + end + end +end -- cgit v1.2.1 From c0b4f8ac3dea446d2b7bdd04eecf3456f79fe7e7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 27 Apr 2018 14:31:26 +0900 Subject: Remove legacy artifacts file_store migration --- .../fill_file_store_build_archive.rb | 21 --------------------- .../fill_file_store_build_metadata.rb | 21 --------------------- 2 files changed, 42 deletions(-) delete mode 100644 lib/gitlab/background_migration/fill_file_store_build_archive.rb delete mode 100644 lib/gitlab/background_migration/fill_file_store_build_metadata.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/fill_file_store_build_archive.rb b/lib/gitlab/background_migration/fill_file_store_build_archive.rb deleted file mode 100644 index 02d744b7a74..00000000000 --- a/lib/gitlab/background_migration/fill_file_store_build_archive.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/AbcSize -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillFileStoreBuildArchive - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - end - - def perform(start_id, stop_id) - FillFileStoreBuildArchive::Build - .where('artifacts_file_store = NULL') - .where(id: (start_id..stop_id)) - .update_all(artifacts_file_store: 1) - end - end - end -end diff --git a/lib/gitlab/background_migration/fill_file_store_build_metadata.rb b/lib/gitlab/background_migration/fill_file_store_build_metadata.rb deleted file mode 100644 index bf1a10b104e..00000000000 --- a/lib/gitlab/background_migration/fill_file_store_build_metadata.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/AbcSize -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillFileStoreBuildMetadata - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - end - - def perform(start_id, stop_id) - FillFileStoreBuildMetadata::Build - .where('artifacts_metadata_store = NULL') - .where(id: (start_id..stop_id)) - .update_all(artifacts_metadata_store: 1) - end - end - end -end -- cgit v1.2.1 From a366fa0988adfe05028395249fc95b967d4fa423 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 27 Apr 2018 15:38:14 +0900 Subject: Add spec for fill_file_store migration --- lib/gitlab/background_migration/fill_file_store_job_artifact.rb | 4 ++-- lib/gitlab/background_migration/fill_file_store_lfs_object.rb | 4 ++-- lib/gitlab/background_migration/fill_file_store_upload.rb | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb index dfd1e7191d2..508e0102911 100644 --- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -10,8 +10,8 @@ module Gitlab end def perform(start_id, stop_id) - FillFileStoreJobArtifact::JobArtifact - .where('file_store = NULL') + Gitlab::BackgroundMigration::FillFileStoreJobArtifact::JobArtifact + .where('file_store is NULL') .where(id: (start_id..stop_id)) .update_all(file_store: 1) end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb index d423bcfba95..5cd926b8121 100644 --- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -10,8 +10,8 @@ module Gitlab end def perform(start_id, stop_id) - FillFileStoreLfsObject::LfsObject - .where('file_store = NULL') + Gitlab::BackgroundMigration::FillFileStoreLfsObject::LfsObject + .where('file_store is NULL') .where(id: (start_id..stop_id)) .update_all(file_store: 1) end diff --git a/lib/gitlab/background_migration/fill_file_store_upload.rb b/lib/gitlab/background_migration/fill_file_store_upload.rb index 14f4c667ed8..203c3cea09f 100644 --- a/lib/gitlab/background_migration/fill_file_store_upload.rb +++ b/lib/gitlab/background_migration/fill_file_store_upload.rb @@ -11,8 +11,8 @@ module Gitlab end def perform(start_id, stop_id) - FillFileStoreUpload::Upload - .where('store = NULL') + Gitlab::BackgroundMigration::FillFileStoreUpload::Upload + .where('store is NULL') .where(id: (start_id..stop_id)) .update_all(store: 1) end -- cgit v1.2.1 From 321e580eded961d4de1570d2dd24e8140b0e1b77 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 18 May 2018 17:04:05 +0900 Subject: Rename FillFileStoreUpload to FillStoreUpload --- .../background_migration/fill_file_store_upload.rb | 21 --------------------- .../background_migration/fill_store_upload.rb | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 lib/gitlab/background_migration/fill_file_store_upload.rb create mode 100644 lib/gitlab/background_migration/fill_store_upload.rb (limited to 'lib') diff --git a/lib/gitlab/background_migration/fill_file_store_upload.rb b/lib/gitlab/background_migration/fill_file_store_upload.rb deleted file mode 100644 index 203c3cea09f..00000000000 --- a/lib/gitlab/background_migration/fill_file_store_upload.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/AbcSize -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillFileStoreUpload - class Upload < ActiveRecord::Base - self.table_name = 'uploads' - self.inheritance_column = :_type_disabled - end - - def perform(start_id, stop_id) - Gitlab::BackgroundMigration::FillFileStoreUpload::Upload - .where('store is NULL') - .where(id: (start_id..stop_id)) - .update_all(store: 1) - end - end - end -end diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb new file mode 100644 index 00000000000..669a9351fb4 --- /dev/null +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FillStoreUpload + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + Gitlab::BackgroundMigration::FillStoreUpload::Upload + .where('store is NULL') + .where(id: (start_id..stop_id)) + .update_all(store: 1) + end + end + end +end -- cgit v1.2.1 From 2eaef5b3a9da09764c74a6f375c8245a5f9a4892 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 22 May 2018 15:36:59 +0900 Subject: Remove Gitlab::BackgroundMigration exxxplicit namespaces from migration classes --- lib/gitlab/background_migration/fill_file_store_job_artifact.rb | 4 ++-- lib/gitlab/background_migration/fill_file_store_lfs_object.rb | 4 ++-- lib/gitlab/background_migration/fill_store_upload.rb | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb index 508e0102911..22b0ac71920 100644 --- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -10,8 +10,8 @@ module Gitlab end def perform(start_id, stop_id) - Gitlab::BackgroundMigration::FillFileStoreJobArtifact::JobArtifact - .where('file_store is NULL') + FillFileStoreJobArtifact::JobArtifact + .where(file_store: nil) .where(id: (start_id..stop_id)) .update_all(file_store: 1) end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb index 5cd926b8121..d0816ae3ed5 100644 --- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -10,8 +10,8 @@ module Gitlab end def perform(start_id, stop_id) - Gitlab::BackgroundMigration::FillFileStoreLfsObject::LfsObject - .where('file_store is NULL') + FillFileStoreLfsObject::LfsObject + .where(file_store: nil) .where(id: (start_id..stop_id)) .update_all(file_store: 1) end diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb index 669a9351fb4..94c65459a67 100644 --- a/lib/gitlab/background_migration/fill_store_upload.rb +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -11,8 +11,8 @@ module Gitlab end def perform(start_id, stop_id) - Gitlab::BackgroundMigration::FillStoreUpload::Upload - .where('store is NULL') + FillStoreUpload::Upload + .where(store: nil) .where(id: (start_id..stop_id)) .update_all(store: 1) end -- cgit v1.2.1 From b529d04b69c5fd8612bfa614a536d02574cb7f07 Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Mon, 28 May 2018 09:58:14 +0000 Subject: Use Gitaly's DeleteAllRepositories RPC during backup restore --- lib/backup/repository.rb | 56 ++++++++++++++++++----------- lib/gitlab/gitaly_client/storage_service.rb | 15 ++++++++ 2 files changed, 51 insertions(+), 20 deletions(-) create mode 100644 lib/gitlab/gitaly_client/storage_service.rb (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index c3360c391af..84670d6582e 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -73,29 +73,40 @@ module Backup end def prepare_directories - # TODO: Need to find a way to do this for gitaly - # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194 - Gitlab.config.repositories.storages.each do |name, repository_storage| - path = repository_storage.legacy_disk_path - next unless File.exist?(path) - - # Move all files in the existing repos directory except . and .. to - # repositories.old. directory - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] - - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) + delete_all_repositories(name, repository_storage) + end + end + + def delete_all_repositories(name, repository_storage) + gitaly_migrate(:delete_all_repositories) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories + else + local_delete_all_repositories(name, repository_storage) end end end + def local_delete_all_repositories(name, repository_storage) + path = repository_storage.legacy_disk_path + return unless File.exist?(path) + + # Move all files in the existing repos directory except . and .. to + # repositories.old. directory + bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) + FileUtils.mkdir_p(bk_repos_path, mode: 0700) + files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + + begin + FileUtils.mv(files, bk_repos_path) + rescue Errno::EACCES + access_denied_error(path) + rescue Errno::EBUSY + resource_busy_error(path) + end + end + def restore_custom_hooks(project) # TODO: Need to find a way to do this for gitaly # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 @@ -113,6 +124,7 @@ module Backup def restore prepare_directories gitlab_shell = Gitlab::Shell.new + Project.find_each(batch_size: 1000) do |project| progress.print " * #{project.full_path} ... " path_to_project_bundle = path_to_bundle(project) @@ -121,7 +133,6 @@ module Backup restore_repo_success = nil if File.exist?(path_to_project_bundle) begin - gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists? project.repository.create_from_bundle path_to_project_bundle restore_repo_success = true rescue => e @@ -146,7 +157,6 @@ module Backup if File.exist?(path_to_wiki_bundle) progress.print " * #{wiki.full_path} ... " begin - gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists? wiki.repository.create_from_bundle(path_to_wiki_bundle) progress.puts "[DONE]".color(:green) rescue => e @@ -224,5 +234,11 @@ module Backup def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end + + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) + Gitlab::GitalyClient.migrate(method, status: status, &block) + rescue GRPC::NotFound, GRPC::BadStatus => e + raise Error, e + end end end diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb new file mode 100644 index 00000000000..eb0e910665b --- /dev/null +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -0,0 +1,15 @@ +module Gitlab + module GitalyClient + class StorageService + def initialize(storage) + @storage = storage + end + + # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. + def delete_all_repositories + request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) + GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request) + end + end + end +end -- cgit v1.2.1 From 954968c684d12a58b265c9e3796faddc84edd400 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 15 May 2018 14:47:38 +0200 Subject: Eager load project routes for issue references This reduces the number of queries necessary when displaying the activity page of a group. --- lib/banzai/reference_parser/issue_parser.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 6bee5ea15b9..7b5915899cf 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -69,7 +69,8 @@ module Banzai { group: [:owners, :group_members] }, :invited_groups, :project_members, - :project_feature + :project_feature, + :route ] } ), -- cgit v1.2.1 From d6e3a6b5fb79ae1983daf096bf87fe14d298e6dc Mon Sep 17 00:00:00 2001 From: Davin Walker Date: Mon, 28 May 2018 09:57:17 -0600 Subject: include groups in description --- lib/api/settings.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/settings.rb b/lib/api/settings.rb index e31c332b6e4..d727ad59367 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -24,7 +24,7 @@ module API optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' - optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' -- cgit v1.2.1 From 16e04e006cf8996883ef0c5b7881eb3cbb7079ce Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 25 May 2018 21:00:26 -0700 Subject: Log queue duration for Grape API calls This mimics the same thing the RailsQueueDuration does for Rails controller requests and will help diagnose issues with API delays. Closes #46601 --- lib/api/api.rb | 3 ++- .../grape_logging/loggers/queue_duration_logger.rb | 26 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/grape_logging/loggers/queue_duration_logger.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index de20b2b8e67..206fabe5c43 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -15,7 +15,8 @@ module API include: [ GrapeLogging::Loggers::FilterParameters.new, GrapeLogging::Loggers::ClientEnv.new, - Gitlab::GrapeLogging::Loggers::UserLogger.new + Gitlab::GrapeLogging::Loggers::UserLogger.new, + Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new ] allow_access_with_scope :api diff --git a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb new file mode 100644 index 00000000000..0adac79f25a --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb @@ -0,0 +1,26 @@ +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log how much time an API request was queued by Workhorse. +module Gitlab + module GrapeLogging + module Loggers + class QueueDurationLogger < ::GrapeLogging::Loggers::Base + attr_accessor :start_time + + def before + @start_time = Time.now + end + + def parameters(request, _) + proxy_start = request.env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence + + return {} unless proxy_start && start_time + + # Time in milliseconds since gitlab-workhorse started the request + duration = (start_time.to_f * 1_000 - proxy_start.to_f / 1_000_000).round(2) + + { 'queue_duration': duration } + end + end + end + end +end -- cgit v1.2.1