diff options
Diffstat (limited to 'lib')
43 files changed, 572 insertions, 807 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 13cfba728fa..4b223a391ae 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -45,6 +45,7 @@ module API present( paginate(::Kaminari.paginate_array(branches)), with: Entities::Branch, + current_user: current_user, project: user_project, merged_branch_names: merged_branch_names ) @@ -63,7 +64,7 @@ module API get do branch = find_branch!(params[:branch]) - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project end end @@ -101,7 +102,7 @@ module API end if protected_branch.valid? - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project else render_api_error!(protected_branch.errors.full_messages, 422) end @@ -121,7 +122,7 @@ module API protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch&.destroy - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project end desc 'Create branch' do @@ -140,6 +141,7 @@ module API if result[:status] == :success present result[:branch], with: Entities::Branch, + current_user: current_user, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/files.rb b/lib/api/files.rb index 1598d3c00b8..29d7489bd7c 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -5,6 +5,8 @@ module API # Prevents returning plain/text responses for files with .txt extension after_validation { content_type "application/json" } + helpers ::API::Helpers::HeadersHelpers + helpers do def commit_params(attrs) { @@ -40,6 +42,20 @@ module API } end + def blob_data + { + file_name: @blob.name, + file_path: @blob.path, + size: @blob.size, + encoding: "base64", + content_sha256: Digest::SHA256.hexdigest(@blob.data), + ref: params[:ref], + blob_id: @blob.id, + commit_id: @commit.id, + last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) + } + end + params :simple_file_params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' @@ -61,6 +77,17 @@ module API requires :id, type: String, desc: 'The project ID' end resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do + desc 'Get raw file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit' + end + head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' @@ -69,9 +96,22 @@ module API get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! + set_http_headers(blob_data) + send_git_blob @repo, @blob end + desc 'Get file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit' + end + head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + desc 'Get a file from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' @@ -80,17 +120,11 @@ module API get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! - { - file_name: @blob.name, - file_path: @blob.path, - size: @blob.size, - encoding: "base64", - content: Base64.strict_encode64(@blob.data), - ref: params[:ref], - blob_id: @blob.id, - commit_id: @commit.id, - last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) - } + data = blob_data + + set_http_headers(data) + + data.merge(content: Base64.strict_encode64(@blob.data)) end desc 'Create new file in repository' diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb new file mode 100644 index 00000000000..cde51fccc62 --- /dev/null +++ b/lib/api/helpers/headers_helpers.rb @@ -0,0 +1,11 @@ +module API + module Helpers + module HeadersHelpers + def set_http_headers(header_data) + header_data.each do |key, value| + header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value + end + end + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index af7d2471b34..0f46bc4c98e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -72,8 +72,8 @@ module API end params :merge_requests_params do - optional :state, type: String, values: %w[opened closed merged all], default: 'all', - desc: 'Return opened, closed, merged, or all merge requests' + optional :state, type: String, values: %w[opened closed locked merged all], default: 'all', + desc: 'Return opened, closed, locked, merged, or all merge requests' optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3ef3680c5d9..b83da00502d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -459,6 +459,23 @@ module API conflict!(error.message) end end + + desc 'Transfer a project to a new namespace' + params do + requires :namespace, type: String, desc: 'The ID or path of the new namespace' + end + put ":id/transfer" do + authorize! :change_namespace, user_project + + namespace = find_namespace!(params[:namespace]) + result = ::Projects::TransferService.new(user_project, current_user).execute(namespace) + + if result + present user_project, with: Entities::Project + else + render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400) + end + end end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index bb3fa99af38..33a9646ac3b 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -100,9 +100,10 @@ module API params do requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false end get ':id/repository/compare' do - compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) + compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to], straight: params[:straight]) present compare, with: Entities::Compare end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 50a5e340191..af762db517c 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -48,7 +48,7 @@ module Backup end def backup_project(project) - gitaly_migrate(:repository_backup) do |is_enabled| + gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled backup_project_gitaly(project) else @@ -80,7 +80,7 @@ module Backup end def delete_all_repositories(name, repository_storage) - gitaly_migrate(:delete_all_repositories) do |is_enabled| + gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories else @@ -148,7 +148,7 @@ module Backup end def backup_custom_hooks(project) - gitaly_migrate(:backup_custom_hooks) do |is_enabled| + gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_backup_custom_hooks(project) else @@ -159,7 +159,7 @@ module Backup def restore_custom_hooks(project) in_path(path_to_tars(project)) do |dir| - gitaly_migrate(:restore_custom_hooks) do |is_enabled| + gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_restore_custom_hooks(project, dir) else diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 4bc82ecb4d6..bb9f488cd87 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -56,10 +56,12 @@ module Banzai # Pattern to match allowed image extensions ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze + # Do not perform linking inside these tags. + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + def call doc.search(".//text()").each do |node| - # Do not perform linking inside <code> blocks - next unless node.ancestors('code').empty? + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running # before this one, it will be converted into `[[<em>TOC</em>]]`, so it diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 6786b9d07b6..afc2ca4e362 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -25,10 +25,11 @@ module Banzai # Only push these customizations once return if customized?(whitelist[:transformers]) - # Allow table alignment; we whitelist specific style properties in a + # Allow table alignment; we whitelist specific text-align values in a # transformer below whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) + whitelist[:css] = { properties: ['text-align'] } # Allow span elements whitelist[:elements].push('span') diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 97244159985..b32660a8341 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -92,7 +92,7 @@ module Banzai def text return '' unless node - @text ||= node.text + @text ||= EscapeUtils.escape_html(node.text) end private diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index a1f24e8b093..0d9b874ef85 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -44,11 +44,7 @@ module Banzai def self.transform_context(context) context[:only_path] = true unless context.key?(:only_path) - context.merge( - # EmojiFilter - asset_host: Gitlab::Application.config.asset_host, - asset_root: Gitlab.config.gitlab.base_url - ) + context end end end diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index 605e93022e7..2760211fee8 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -22,6 +22,18 @@ module Gitaly server_version == Gitlab::GitalyClient.expected_server_version end + def read_writeable? + readable? && writeable? + end + + def readable? + storage_status&.readable + end + + def writeable? + storage_status&.writeable + end + def address Gitlab::GitalyClient.address(@storage) rescue RuntimeError => e @@ -30,13 +42,17 @@ module Gitaly private + def storage_status + @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage } + end + def info @info ||= begin Gitlab::GitalyClient::ServerService.new(@storage).info rescue GRPC::Unavailable, GRPC::GRPC::DeadlineExceeded # This will show the server as being out of date - Gitaly::ServerInfoResponse.new(git_version: '', server_version: '') + Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) end end end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 6c5d0788a0a..e7283b2f9e8 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -74,6 +74,10 @@ module Gitlab gl_user end + def bypass_two_factor? + false + end + protected def should_save? diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index c345a7e3f6c..3bc5e2864df 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -6,6 +6,17 @@ module Gitlab Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) end + def authn_context + response_object = auth_hash.extra[:response_object] + return nil if response_object.blank? + + document = response_object.decrypted_document + document ||= response_object.document + return nil if document.blank? + + extract_authn_context(document) + end + private def get_raw(key) @@ -13,6 +24,10 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end + + def extract_authn_context(document) + REXML::XPath.first(document, "//saml:AuthnStatement/saml:AuthnContext/saml:AuthnContextClassRef/text()").to_s + end end end end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 5fa9581f837..625dab7c6f4 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -7,6 +7,10 @@ module Gitlab Gitlab::Auth::OAuth::Provider.config_for('saml') end + def upstream_two_factor_authn_contexts + options.args[:upstream_two_factor_authn_contexts] + end + def groups options[:groups_attribute] end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index b8c84c37cd5..6c3b75f3eb0 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -34,6 +34,10 @@ module Gitlab gl_user.changed? || gl_user.identities.any?(&:changed?) end + def bypass_two_factor? + saml_config.upstream_two_factor_authn_contexts&.include?(auth_hash.authn_context) + end + protected def saml_config diff --git a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb new file mode 100644 index 00000000000..d3f366f3480 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for cleaning up a concurrent column rename. + class CleanupConcurrentRename < CleanupConcurrentSchemaChange + RESCHEDULE_DELAY = 10.minutes + + def cleanup_concurrent_schema_change(table, old_column, new_column) + cleanup_concurrent_column_rename(table, old_column, new_column) + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb new file mode 100644 index 00000000000..54f77f184d5 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Base class for cleaning up concurrent schema changes. + class CleanupConcurrentSchemaChange + include Database::MigrationHelpers + + # table - The name of the table the migration is performed for. + # old_column - The name of the old (to drop) column. + # new_column - The name of the new column. + def perform(table, old_column, new_column) + return unless column_exists?(table, new_column) + + rows_to_migrate = define_model_for(table) + .where(new_column => nil) + .where + .not(old_column => nil) + + if rows_to_migrate.any? + BackgroundMigrationWorker.perform_in( + RESCHEDULE_DELAY, + self.class.name, + [table, old_column, new_column] + ) + else + cleanup_concurrent_schema_change(table, old_column, new_column) + end + end + + # These methods are necessary so we can re-use the migration helpers in + # this class. + def connection + ActiveRecord::Base.connection + end + + def method_missing(name, *args, &block) + connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(*args) + connection.respond_to?(*args) || super + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb index de622f657b2..48411095dbb 100644 --- a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb +++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb @@ -2,52 +2,12 @@ module Gitlab module BackgroundMigration - # Background migration for cleaning up a concurrent column rename. - class CleanupConcurrentTypeChange - include Database::MigrationHelpers - + # Background migration for cleaning up a concurrent column type changeb. + class CleanupConcurrentTypeChange < CleanupConcurrentSchemaChange RESCHEDULE_DELAY = 10.minutes - # table - The name of the table the migration is performed for. - # old_column - The name of the old (to drop) column. - # new_column - The name of the new column. - def perform(table, old_column, new_column) - return unless column_exists?(:issues, new_column) - - rows_to_migrate = define_model_for(table) - .where(new_column => nil) - .where - .not(old_column => nil) - - if rows_to_migrate.any? - BackgroundMigrationWorker.perform_in( - RESCHEDULE_DELAY, - 'CleanupConcurrentTypeChange', - [table, old_column, new_column] - ) - else - cleanup_concurrent_column_type_change(table, old_column) - end - end - - # These methods are necessary so we can re-use the migration helpers in - # this class. - def connection - ActiveRecord::Base.connection - end - - def method_missing(name, *args, &block) - connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend - end - - def respond_to_missing?(*args) - connection.respond_to?(*args) || super - end - - def define_model_for(table) - Class.new(ActiveRecord::Base) do - self.table_name = table - end + def cleanup_concurrent_schema_change(table, old_column, new_column) + cleanup_concurrent_column_type_change(table, old_column) end end end diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb new file mode 100644 index 00000000000..0b785e1b056 --- /dev/null +++ b/lib/gitlab/background_migration/delete_diff_files.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class DeleteDiffFiles + def perform(merge_request_diff_id) + merge_request_diff = MergeRequestDiff.find_by(id: merge_request_diff_id) + + return unless merge_request_diff + return unless should_delete_diff_files?(merge_request_diff) + + MergeRequestDiff.transaction do + merge_request_diff.update_column(:state, 'without_files') + + # explain (analyze, buffers) when deleting 453 diff files: + # + # Delete on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actual time=43.265..43.265 rows=0 loops=1) + # Buffers: shared hit=2043 read=259 dirtied=254 + # -> Index Scan using index_merge_request_diff_files_on_mr_diff_id_and_order on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actu + # al time=0.466..26.317 rows=453 loops=1) + # Index Cond: (merge_request_diff_id = 463448) + # Buffers: shared hit=17 read=84 + # Planning time: 0.107 ms + # Execution time: 43.287 ms + # + MergeRequestDiffFile.where(merge_request_diff_id: merge_request_diff.id).delete_all + end + end + + private + + def should_delete_diff_files?(merge_request_diff) + return false if merge_request_diff.state == 'without_files' + + merge_request = merge_request_diff.merge_request + + return false unless merge_request.state == 'merged' + return false if merge_request_diff.id == merge_request.latest_merge_request_diff_id + + true + end + end + end +end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index ecc85f847d4..671b8e7e1b1 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -1,41 +1,6 @@ module Gitlab module Cache - # This module provides a simple way to cache values in RequestStore, - # and the cache key would be based on the class name, method name, - # optionally customized instance level values, optionally customized - # method level values, and optional method arguments. - # - # A simple example: - # - # class UserAccess - # extend Gitlab::Cache::RequestCache - # - # request_cache_key do - # [user&.id, project&.id] - # end - # - # request_cache def can_push_to_branch?(ref) - # # ... - # end - # end - # - # This way, the result of `can_push_to_branch?` would be cached in - # `RequestStore.store` based on the cache key. If RequestStore is not - # currently active, then it would be stored in a hash saved in an - # instance variable, so the cache logic would be the same. - # Here's another example using customized method level values: - # - # class Commit - # extend Gitlab::Cache::RequestCache - # - # def author - # User.find_by_any_email(author_email.downcase) - # end - # request_cache(:author) { author_email.downcase } - # end - # - # So that we could have different strategies for different methods - # + # See https://docs.gitlab.com/ee/development/utilities.html#requestcache module RequestCache def self.extended(klass) return if klass < self diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index d00e5b07f95..222aa06b800 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -4,6 +4,9 @@ module Gitlab class Collection class Item def initialize(key:, value:, public: true, file: false) + raise ArgumentError, "`value` must be of type String, while it was: #{value.class}" unless + value.is_a?(String) || value.nil? + @variable = { key: key, value: value, public: public, file: file } diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 3cac007a42c..f64e3d53138 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -33,7 +33,13 @@ module Gitlab end def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table + arel_from = if Gitlab.rails5? + arel_table.from + else + arel_table + end + + query = arel_from .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) .project(average([arel_table[column_sym]], 'median')) .where( diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index c21bae5e16b..4fe5b4cc835 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -596,6 +596,97 @@ module Gitlab end end + # Renames a column using a background migration. + # + # Because this method uses a background migration it's more suitable for + # large tables. For small tables it's better to use + # `rename_column_concurrently` since it can complete its work in a much + # shorter amount of time and doesn't rely on Sidekiq. + # + # Example usage: + # + # rename_column_using_background_migration( + # :users, + # :feed_token, + # :rss_token + # ) + # + # table - The name of the database table containing the column. + # + # old - The old column name. + # + # new - The new column name. + # + # type - The type of the new column. If no type is given the old column's + # type is used. + # + # batch_size - The number of rows to schedule in a single background + # migration. + # + # interval - The time interval between every background migration. + def rename_column_using_background_migration( + table, + old_column, + new_column, + type: nil, + batch_size: 10_000, + interval: 10.minutes + ) + + check_trigger_permissions!(table) + + old_col = column_for(table, old_column) + new_type = type || old_col.type + max_index = 0 + + add_column(table, new_column, new_type, + limit: old_col.limit, + precision: old_col.precision, + scale: old_col.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new_column, old_col.default) if old_col.default + + install_rename_triggers(table, old_column, new_column) + + model = Class.new(ActiveRecord::Base) do + self.table_name = table + + include ::EachBatch + end + + # Schedule the jobs that will copy the data from the old column to the + # new one. Rows with NULL values in our source column are skipped since + # the target column is already NULL at this point. + model.where.not(old_column => nil).each_batch(of: batch_size) do |batch, index| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + max_index = index + + BackgroundMigrationWorker.perform_in( + index * interval, + 'CopyColumn', + [table, old_column, new_column, start_id, end_id] + ) + end + + # Schedule the renaming of the column to happen (initially) 1 hour after + # the last batch finished. + BackgroundMigrationWorker.perform_in( + (max_index * interval) + 1.hour, + 'CleanupConcurrentRename', + [table, old_column, new_column] + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentRename') + end + end + def perform_background_migration_inline? Rails.env.test? || Rails.env.development? end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index d512fc58e46..4850a6c0430 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -38,7 +38,8 @@ module Gitlab # we only want to create full urls when there's a different asset_host # configured. def host - if Gitlab::Application.config.asset_host.nil? || Gitlab::Application.config.asset_host == Gitlab.config.gitlab.base_url + asset_host = ActionController::Base.asset_host + if asset_host.nil? || asset_host == Gitlab.config.gitlab.base_url nil else Gitlab.config.gitlab.base_url diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 156d077a69c..604bb11e712 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -21,13 +21,31 @@ module Gitlab attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary class << self - def find(repository, sha, path) - Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled| - if is_enabled - find_by_gitaly(repository, sha, path) - else - find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - end + def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + + path = path.sub(%r{\A/*}, '') + path = '/' if path.empty? + name = File.basename(path) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) + return unless entry + + entry.data = "" if limit == 0 + + case entry.type + when :COMMIT + new(id: entry.oid, name: name, size: 0, data: '', path: path, commit_id: sha) + when :BLOB + new(id: entry.oid, name: name, size: entry.size, data: entry.data.dup, mode: entry.mode.to_s(8), + path: path, commit_id: sha, binary: binary?(entry.data)) end end @@ -56,7 +74,7 @@ module Gitlab repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a else blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + find(repository, sha, path, limit: blob_size_limit) end end end @@ -136,85 +154,6 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - return unless path - - path = path.sub(%r{\A/*}, '') - path = '/' if path.empty? - name = File.basename(path) - - # Gitaly will think that setting the limit to 0 means unlimited, while - # the client might only need the metadata and thus set the limit to 0. - # In this method we'll then set the limit to 1, but clear the byte of data - # that we got back so for the outside world it looks like the limit was - # actually 0. - req_limit = limit == 0 ? 1 : limit - - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) - return unless entry - - entry.data = "" if limit == 0 - - case entry.type - when :COMMIT - new( - id: entry.oid, - name: name, - size: 0, - data: '', - path: path, - commit_id: sha - ) - when :BLOB - new( - id: entry.oid, - name: name, - size: entry.size, - data: entry.data.dup, - mode: entry.mode.to_s(8), - path: path, - commit_id: sha, - binary: binary?(entry.data) - ) - end - end - - def find_by_rugged(repository, sha, path, limit:) - return unless path - - # Strip any leading / characters from the path - path = path.sub(%r{\A/*}, '') - - rugged_commit = repository.lookup(sha) - root_tree = rugged_commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) - - return nil unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - # Rugged::Blob#content is expensive; don't call it if we don't have to. - data: limit.zero? ? '' : blob.content(limit), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) - end - end - rescue Rugged::ReferenceError - nil - end - def rugged_raw(repository, sha, limit:) blob = repository.lookup(sha) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index c9806cdb85f..c67826da1d2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -116,15 +116,9 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.between(base, head) - else - repo.rugged_commits_between(base, head).map { |c| decorate(repo, c) } - end + repo.wrapped_gitaly_errors do + repo.gitaly_commit_client.between(base, head) end - rescue Rugged::ReferenceError - [] end # Returns commits collection @@ -149,56 +143,9 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) - Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled| - if is_enabled - find_all_by_gitaly(repo, options) - else - find_all_by_rugged(repo, options) - end - end - end - - def find_all_by_rugged(repo, options = {}) - actual_options = options.dup - - allowed_options = [:ref, :max_count, :skip, :order] - - actual_options.keep_if do |key| - allowed_options.include?(key) + repo.wrapped_gitaly_errors do + Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end - - default_options = { skip: 0 } - actual_options = default_options.merge(actual_options) - - rugged = repo.rugged - walker = Rugged::Walker.new(rugged) - - if actual_options[:ref] - walker.push(rugged.rev_parse_oid(actual_options[:ref])) - else - rugged.references.each("refs/heads/*") do |ref| - walker.push(ref.target_id) - end - end - - walker.sorting(rugged_sort_type(actual_options[:order])) - - commits = [] - offset = actual_options[:skip] - limit = actual_options[:max_count] - walker.each(offset: offset, limit: limit) do |commit| - commits.push(decorate(repo, commit)) - end - - walker.reset - - commits - rescue Rugged::OdbError - [] - end - - def find_all_by_gitaly(repo, options = {}) - Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end def decorate(repository, commit, ref = nil) @@ -220,19 +167,7 @@ module Gitlab end def shas_with_signatures(repository, shas) - GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) - else - shas.select do |sha| - begin - Rugged::Commit.extract_signature(repository.rugged, sha) - rescue Rugged::OdbError - false - end - end - end - end + Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) end # Only to be used when the object ids will not necessarily have a @@ -250,13 +185,7 @@ module Gitlab end def extract_signature(repository, commit_id) - repository.gitaly_migrate(:extract_commit_signature) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.extract_signature(commit_id) - else - rugged_extract_signature(repository, commit_id) - end - end + repository.gitaly_commit_client.extract_signature(commit_id) end def extract_signature_lazily(repository, commit_id) @@ -276,36 +205,9 @@ module Gitlab end def batch_signature_extraction(repository, commit_ids) - repository.gitaly_migrate(:extract_commit_signature_in_batch) do |is_enabled| - if is_enabled - gitaly_batch_signature_extraction(repository, commit_ids) - else - rugged_batch_signature_extraction(repository, commit_ids) - end - end - end - - def gitaly_batch_signature_extraction(repository, commit_ids) repository.gitaly_commit_client.get_commit_signatures(commit_ids) end - def rugged_batch_signature_extraction(repository, commit_ids) - commit_ids.each_with_object({}) do |commit_id, signatures| - signature_data = rugged_extract_signature(repository, commit_id) - next unless signature_data - - signatures[commit_id] = signature_data - end - end - - def rugged_extract_signature(repository, commit_id) - begin - Rugged::Commit.extract_signature(repository.rugged, commit_id) - rescue Rugged::OdbError - nil - end - end - def get_message(repository, commit_id) BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| items_by_repo = items.group_by { |i| i[:repository] } @@ -323,13 +225,7 @@ module Gitlab end def get_messages(repository, commit_ids) - repository.gitaly_migrate(:commit_messages) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.get_commit_messages(commit_ids) - else - commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h - end - end + repository.gitaly_commit_client.get_commit_messages(commit_ids) end end @@ -381,15 +277,11 @@ module Gitlab # empty repo. See Repository#diff for keys allowed in the +options+ # hash. def diff_from_parent(options = {}) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.diff_from_parent(self, options) - else - rugged_diff_from_parent(options) - end - end + @repository.gitaly_commit_client.diff_from_parent(self, options) end + # Not to be called directly, but right now its used for tests and in old + # migrations def rugged_diff_from_parent(options = {}) options ||= {} break_rewrites = options[:break_rewrites] @@ -497,13 +389,18 @@ module Gitlab def tree_entry(path) return unless path.present? - @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| - if is_migrated - gitaly_tree_entry(path) - else - rugged_tree_entry(path) - end - end + # We're only interested in metadata, so limit actual data to 1 byte + # since Gitaly doesn't support "send no data" option. + entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) + return unless entry + + # To be compatible with the rugged format + entry = entry.to_h + entry.delete(:data) + entry[:name] = File.basename(path) + entry[:type] = entry[:type].downcase + + entry end def to_gitaly_commit @@ -566,28 +463,6 @@ module Gitlab SERIALIZE_KEYS end - def gitaly_tree_entry(path) - # We're only interested in metadata, so limit actual data to 1 byte - # since Gitaly doesn't support "send no data" option. - entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) - return unless entry - - # To be compatible with the rugged format - entry = entry.to_h - entry.delete(:data) - entry[:name] = File.basename(path) - entry[:type] = entry[:type].downcase - - entry - end - - # Is this the same as Blob.find_entry_by_path ? - def rugged_tree_entry(path) - rugged_commit.tree.path(path) - rescue Rugged::TreeError - nil - end - def gitaly_commit_author_from_rugged(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index 8475645971e..5ff15a787f0 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -61,22 +61,15 @@ module Gitlab end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - tags_option = tags ? '--tags' : '--no-tags' - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet) - cmd << '--prune' if prune - cmd << '--force' if force - cmd << tags_option + cmd = fetch_remote_command(name, tags, prune, force) setup_ssh_auth(ssh_key, known_hosts) do |env| - success = run_with_timeout(cmd, timeout, repository_absolute_path, env) - - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| + unless success + logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + end end - - success end end @@ -202,6 +195,14 @@ module Gitlab private + def fetch_remote_command(name, tags, prune, force) + %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| + cmd << '--prune' if prune + cmd << '--force' if force + cmd << (tags ? '--tags' : '--no-tags') + end + end + def git_import_repository(source, timeout) # Skip import if repo already exists return false if File.exist?(repository_absolute_path) diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index ebe46722890..e4743b4db0a 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -7,81 +7,8 @@ module Gitlab end def update(only_branches_matching: []) - @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| - if is_enabled - gitaly_update(only_branches_matching) - else - rugged_update(only_branches_matching) - end - end - end - - private - - def gitaly_update(only_branches_matching) - @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) - end - - def rugged_update(only_branches_matching) - local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) - remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) - - updated_branches = changed_refs(local_branches, remote_branches) - push_branches(updated_branches.keys) if updated_branches.present? - - delete_refs(local_branches, remote_branches) - - local_tags = refs_obj(@repository.tags) - remote_tags = refs_obj(@repository.remote_tags(@ref_name)) - - updated_tags = changed_refs(local_tags, remote_tags) - @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? - - delete_refs(local_tags, remote_tags) - end - - def refs_obj(refs, only_refs_matching: []) - refs.each_with_object({}) do |ref, refs| - next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) - - refs[ref.name] = ref - end - end - - def changed_refs(local_refs, remote_refs) - local_refs.select do |ref_name, ref| - remote_ref = remote_refs[ref_name] - - remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target - end - end - - def push_branches(branches) - default_branch, branches = branches.partition do |branch| - @repository.root_ref == branch - end - - # Push the default branch first so it works fine when remote mirror is empty. - branches.unshift(*default_branch) - - @repository.push_remote_branches(@ref_name, branches) - end - - def delete_refs(local_refs, remote_refs) - refs = refs_to_delete(local_refs, remote_refs) - - @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? - end - - def refs_to_delete(local_refs, remote_refs) - default_branch_id = @repository.commit.id - - remote_refs.select do |remote_ref_name, remote_ref| - next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo - - remote_ref_id = remote_ref.dereferenced_target.try(:id) - - remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + @repository.wrapped_gitaly_errors do + @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7056d9c8756..7c3b91f6efb 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -403,13 +403,7 @@ module Gitlab # Return repo size in megabytes def size - size = gitaly_migrate(:repository_size) do |is_enabled| - if is_enabled - size_by_gitaly - else - size_by_shelling_out - end - end + size = gitaly_repository_client.repository_size (size.to_f / 1024).round(2) end @@ -498,27 +492,6 @@ module Gitlab Ref.dereference_object(obj) end - # Return a collection of Rugged::Commits between the two revspec arguments. - # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for - # a detailed list of valid arguments. - # - # Gitaly note: JV: to be deprecated in favor of Commit.between - def rugged_commits_between(from, to) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE) - - sha_from = sha_from_ref(from) - sha_to = sha_from_ref(to) - - walker.push(sha_to) - walker.hide(sha_from) - - commits = walker.to_a - walker.reset - - commits - end - # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to, options = {}) count_commits(from: from, to: to, **options) @@ -555,24 +528,9 @@ module Gitlab end end - # Gitaly note: JV: check gitlab-ee before removing this method. - def rugged_is_ancestor?(ancestor_id, descendant_id) - return false if ancestor_id.nil? || descendant_id.nil? - - rugged_merge_base(ancestor_id, descendant_id) == ancestor_id - rescue Rugged::OdbError - false - end - # Returns true is +from+ is direct ancestor to +to+, otherwise false def ancestor?(from, to) - Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| - if is_enabled - gitaly_commit_client.ancestor?(from, to) - else - rugged_is_ancestor?(from, to) - end - end + gitaly_commit_client.ancestor?(from, to) end def merged_branch_names(branch_names = []) @@ -613,17 +571,7 @@ module Gitlab def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? - gitaly_migrate(:find_ref_name) do |is_enabled| - if is_enabled - gitaly_ref_client.find_ref_name(sha, ref_path) - else - args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - run_git(args).first.split.last - end - end + gitaly_ref_client.find_ref_name(sha, ref_path) end # Get refs hash which key is is the commit id @@ -715,6 +663,10 @@ module Gitlab end end + def update_branch(branch_name, user:, newrev:, oldrev:) + OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) + end + def rm_branch(branch_name, user:) gitaly_migrate(:operation_user_delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled @@ -946,13 +898,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) - gitaly_migrate(:ls_files) do |is_enabled| - if is_enabled - gitaly_ls_files(ref) - else - git_ls_files(ref) - end - end + gitaly_commit_client.ls_files(ref) end # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 @@ -1000,29 +946,8 @@ module Gitlab end def languages(ref = nil) - gitaly_migrate(:commit_languages, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_commit_client.languages(ref) - else - ref ||= rugged.head.target_id - languages = Linguist::Repository.new(rugged, ref).languages - total = languages.map(&:last).sum - - languages = languages.map do |language| - name, share = language - color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" - { - value: (share.to_f * 100 / total).round(2), - label: name, - color: color, - highlight: color - } - end - - languages.sort do |x, y| - y[:value] <=> x[:value] - end - end + wrapped_gitaly_errors do + gitaly_commit_client.languages(ref) end end @@ -1180,16 +1105,7 @@ module Gitlab end def create_from_bundle(bundle_path) - gitaly_migrate(:create_repo_from_bundle) do |is_enabled| - if is_enabled - gitaly_repository_client.create_from_bundle(bundle_path) - else - run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil) - self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) - end - end - - true + gitaly_repository_client.create_from_bundle(bundle_path) end def create_from_snapshot(url, auth) @@ -1290,16 +1206,10 @@ module Gitlab return unless full_path.present? # This guard avoids Gitaly log/error spam - unless exists? - raise NoRepository, 'repository does not exist' - end + raise NoRepository, 'repository does not exist' unless exists? - gitaly_migrate(:write_config) do |is_enabled| - if is_enabled - gitaly_repository_client.write_config(full_path: full_path) - else - rugged_write_config(full_path: full_path) - end + wrapped_gitaly_errors do + gitaly_repository_client.write_config(full_path: full_path) end end @@ -1385,16 +1295,7 @@ module Gitlab safe_query = Regexp.escape(query) ref ||= root_ref - gitaly_migrate(:search_files_by_content) do |is_enabled| - if is_enabled - gitaly_repository_client.search_files_by_content(ref, safe_query) - else - offset = 2 - args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref}) - - run_git(args).first.scrub.split(/^--\n/) - end - end + gitaly_repository_client.search_files_by_content(ref, safe_query) end def can_be_merged?(source_sha, target_branch) @@ -1411,15 +1312,7 @@ module Gitlab return [] if empty? || safe_query.blank? - gitaly_migrate(:search_files_by_name) do |is_enabled| - if is_enabled - gitaly_repository_client.search_files_by_name(ref, safe_query) - else - args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query}) - - run_git(args).first.lines.map(&:strip) - end - end + gitaly_repository_client.search_files_by_name(ref, safe_query) end def find_commits_by_message(query, ref, path, limit, offset) @@ -1828,41 +1721,6 @@ module Gitlab commit(sha) end - def size_by_shelling_out - popen(%w(du -sk), path).first.strip.to_i - end - - def size_by_gitaly - gitaly_repository_client.repository_size - end - - def gitaly_ls_files(ref) - gitaly_commit_client.ls_files(ref) - end - - def git_ls_files(ref) - actual_ref = ref || root_ref - - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref}) - raw_output, _status = run_git(cmd) - - lines = raw_output.split("\n").map do |f| - stuff, path = f.split("\t") - _mode, type, _sha = stuff.split(" ") - path if type == "blob" - # Contain only blob type - end - - lines.compact - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -2061,8 +1919,7 @@ module Gitlab rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) + update_branch(branch, user: user, newrev: rebase_sha, oldrev: branch_sha) rebase_sha end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index e44284572fd..bbf2ecdb1fa 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -28,18 +28,7 @@ module Gitlab end def get_messages(repository, tag_ids) - repository.gitaly_migrate(:tag_messages) do |is_enabled| - if is_enabled - repository.gitaly_ref_client.get_tag_messages(tag_ids) - else - tag_ids.map do |id| - tag = repository.rugged.lookup(id) - message = tag.is_a?(Rugged::Commit) ? "" : tag.message - - [id, message] - end.to_h - end - end + repository.gitaly_ref_client.get_tag_messages(tag_ids) end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 7f2e6441f16..c9c414e5d33 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -76,6 +76,13 @@ module Gitlab end def tree_entry(ref, path, limit = nil) + if Pathname.new(path).cleanpath.to_s.start_with?('../') + # The TreeEntry RPC should return an empty reponse in this case but in + # Gitaly 0.107.0 and earlier we get an exception instead. This early return + # saves us a Gitaly roundtrip while also avoiding the exception. + return + end + request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: encode_binary(ref), @@ -317,6 +324,8 @@ module Gitlab return if signature.blank? && signed_text.blank? [signature, signed_text] + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex end def get_commit_signatures(commit_ids) @@ -334,6 +343,8 @@ module Gitlab end signatures + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex end def get_commit_messages(commit_ids) diff --git a/lib/gitlab/graphql/expose_permissions.rb b/lib/gitlab/graphql/expose_permissions.rb new file mode 100644 index 00000000000..e3779995406 --- /dev/null +++ b/lib/gitlab/graphql/expose_permissions.rb @@ -0,0 +1,15 @@ +module Gitlab + module Graphql + module ExposePermissions + extend ActiveSupport::Concern + prepended do + def self.expose_permissions(permission_type, description: 'Permissions for the current user on the resource') + field :user_permissions, permission_type, + description: description, + null: false, + resolve: -> (obj, _, _) { obj } + end + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index 1688262974b..6f2b26c9676 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -10,9 +10,18 @@ module Gitlab old_resolver = field.resolve_proc resolve_with_presenter = -> (presented_type, args, context) do + # We need to wrap the original presentation type into a type that + # uses the presenter as an object. object = presented_type.object + + if object.is_a?(presented_in.presenter_class) + next old_resolver.call(presented_type, args, context) + end + presenter = presented_in.presenter_class.new(object, **context.to_h) - old_resolver.call(presenter, args, context) + wrapped = presented_type.class.new(presenter, context) + + old_resolver.call(wrapped, args, context) end field.redefine do diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb deleted file mode 100644 index 050fe7a5173..00000000000 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ /dev/null @@ -1,169 +0,0 @@ -module Gitlab - module HealthChecks - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1218 - class FsShardsCheck - extend BaseAbstractCheck - RANDOM_STRING = SecureRandom.hex(1000).freeze - COMMAND_TIMEOUT = '1'.freeze - TIMEOUT_EXECUTABLE = 'timeout'.freeze - - class << self - def readiness - repository_storages.map do |storage_name| - begin - if !storage_circuitbreaker_test(storage_name) - HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name) - elsif !storage_stat_test(storage_name) - HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) - else - with_temp_file(storage_name) do |tmp_file_path| - if !storage_write_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) - elsif !storage_read_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) - else - HealthChecks::Result.new(true, nil, shard: storage_name) - end - end - end - rescue RuntimeError => ex - message = "unexpected error #{ex} when checking storage #{storage_name}" - Rails.logger.error(message) - HealthChecks::Result.new(false, message, shard: storage_name) - end - end - end - - def metrics - repository_storages.flat_map do |storage_name| - [ - storage_stat_metrics(storage_name), - storage_write_metrics(storage_name), - storage_read_metrics(storage_name), - storage_circuitbreaker_metrics(storage_name) - ].flatten - end - end - - private - - def operation_metrics(ok_metric, latency_metric, **labels) - result, elapsed = yield - [ - metric(latency_metric, elapsed, **labels), - metric(ok_metric, result ? 1 : 0, **labels) - ] - rescue RuntimeError => ex - Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") - [metric(ok_metric, 0, **labels)] - end - - def repository_storages - storages_paths.keys - end - - def storages_paths - Gitlab.config.repositories.storages - end - - def exec_with_timeout(cmd_args, *args, &block) - Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) - end - - def with_temp_file(storage_name) - temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path } - yield temp_file_path - ensure - delete_test_file(temp_file_path) - end - - def storage_path(storage_name) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - storages_paths[storage_name]&.legacy_disk_path - end - end - - # All below test methods use shell commands to perform actions on storage volumes. - # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely, - # we can rely on shell commands to be terminated once `timeout` kills them. - # - # However we also fallback to pure Ruby file operations in case a specific shell command is missing - # so we are still able to perform healthchecks and gather metrics from such system. - - def delete_test_file(tmp_path) - _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) - status.zero? - rescue Errno::ENOENT - File.delete(tmp_path) rescue Errno::ENOENT - end - - def storage_stat_test(storage_name) - stat_path = File.join(storage_path(storage_name), '.') - begin - _, status = exec_with_timeout(%W{ stat #{stat_path} }) - status.zero? - rescue Errno::ENOENT - File.exist?(stat_path) && File::Stat.new(stat_path).readable? - end - end - - def storage_write_test(tmp_path) - _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| - stdin.write(RANDOM_STRING) - end - status.zero? - rescue Errno::ENOENT - written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT - written_bytes == RANDOM_STRING.length - end - - def storage_read_test(tmp_path) - _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| - stdin.write(RANDOM_STRING) - end - status.zero? - rescue Errno::ENOENT - file_contents = File.read(tmp_path) rescue Errno::ENOENT - file_contents == RANDOM_STRING - end - - def storage_circuitbreaker_test(storage_name) - Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" } - rescue Gitlab::Git::Storage::Inaccessible - nil - end - - def storage_stat_metrics(storage_name) - operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do - with_timing { storage_stat_test(storage_name) } - end - end - - def storage_write_metrics(storage_name) - operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do - with_temp_file(storage_name) do |tmp_file_path| - with_timing { storage_write_test(tmp_file_path) } - end - end - end - - def storage_read_metrics(storage_name) - operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do - with_temp_file(storage_name) do |tmp_file_path| - storage_write_test(tmp_file_path) # writes data used by read test - with_timing { storage_read_test(tmp_file_path) } - end - end - end - - def storage_circuitbreaker_metrics(storage_name) - operation_metrics(:filesystem_circuitbreaker, - :filesystem_circuitbreaker_latency_seconds, - shard: storage_name) do - with_timing { storage_circuitbreaker_test(storage_name) } - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index 11416c002e3..1f623e0b6ec 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -13,14 +13,14 @@ module Gitlab end def metrics - repository_storages.flat_map do |storage_name| - result, elapsed = with_timing { check(storage_name) } - labels = { shard: storage_name } + Gitaly::Server.all.flat_map do |server| + result, elapsed = with_timing { server.read_writeable? } + labels = { shard: server.storage } [ - metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels), + metric("#{metric_prefix}_success", result ? 1 : 0, **labels), metric("#{metric_prefix}_latency_seconds", elapsed, **labels) - ].flatten + ] end end @@ -36,10 +36,6 @@ module Gitlab METRIC_PREFIX end - def successful?(result) - result[:success] - end - def repository_storages storages.keys end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b713fa7e1cd..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.3'.freeze + VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index a39b3bc158c..7b2b3bedf04 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -22,27 +22,27 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) - metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) + metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) + metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:objects_total] = Metrics.gauge(with_prefix(:objects, :total), 'Objects total', labels.merge(class: nil), :livesum) - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :usage_total), 'Memory used total', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors_total), 'File descriptors total', labels, :livesum) + metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end def sample start_time = System.monotonic_time - sample_gc - metrics[:memory_usage].set(labels, System.memory_usage) - metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) + metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) + + sample_gc - metrics[:sampler_duration].observe(labels.merge(worker_label), System.monotonic_time - start_time) + metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) ensure GC::Profiler.clear end @@ -50,11 +50,13 @@ module Gitlab private def sample_gc - metrics[:total_time].set(labels, GC::Profiler.total_time * 1000) - + # Collect generic GC stats. GC.stat.each do |key, value| metrics[key].set(labels, value) end + + # Collect the GC time since last sample in float seconds. + metrics[:total_time].increment(labels, GC::Profiler.total_time) end def worker_label diff --git a/lib/gitlab/shard_health_cache.rb b/lib/gitlab/shard_health_cache.rb new file mode 100644 index 00000000000..3f03f46d4b1 --- /dev/null +++ b/lib/gitlab/shard_health_cache.rb @@ -0,0 +1,41 @@ +module Gitlab + class ShardHealthCache + HEALTHY_SHARDS_KEY = 'gitlab-healthy-shards'.freeze + HEALTHY_SHARDS_TIMEOUT = 300 + + # Clears the Redis set storing the list of healthy shards + def self.clear + Gitlab::Redis::Cache.with { |redis| redis.del(HEALTHY_SHARDS_KEY) } + end + + # Updates the list of healthy shards using a Redis set + # + # shards - An array of shard names to store + def self.update(shards) + Gitlab::Redis::Cache.with do |redis| + redis.multi do |m| + m.del(HEALTHY_SHARDS_KEY) + shards.each { |shard_name| m.sadd(HEALTHY_SHARDS_KEY, shard_name) } + m.expire(HEALTHY_SHARDS_KEY, HEALTHY_SHARDS_TIMEOUT) + end + end + end + + # Returns an array of strings of healthy shards + def self.cached_healthy_shards + Gitlab::Redis::Cache.with { |redis| redis.smembers(HEALTHY_SHARDS_KEY) } + end + + # Checks whether the given shard name is in the list of healthy shards. + # + # shard_name - The string to check + def self.healthy_shard?(shard_name) + Gitlab::Redis::Cache.with { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) } + end + + # Returns the number of healthy shards in the Redis set + def self.healthy_shard_count + Gitlab::Redis::Cache.with { |redis| redis.scard(HEALTHY_SHARDS_KEY) } + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index 1a817eb735b..81f7cd3ffe8 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -15,7 +15,7 @@ module Gitlab if @resource ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})." else - ":sweat_smile: Couldn't identify you, nor can I autorize you!" + ":sweat_smile: Couldn't identify you, nor can I authorize you!" end ephemeral_response(text: message) diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb new file mode 100644 index 00000000000..64634f789da --- /dev/null +++ b/lib/mysql_zero_date.rb @@ -0,0 +1,18 @@ +# Disable NO_ZERO_DATE mode for mysql in rails 5. +# We use zero date as a default value +# (config/initializers/active_record_mysql_timestamp.rb), in +# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) +# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode +# is not sufficient. + +require 'active_record/connection_adapters/abstract_mysql_adapter' + +module MysqlZeroDate + def configure_connection + super + + @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables + end +end + +ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) if Gitlab.rails5? diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 139ab70e125..69166851816 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -46,7 +46,9 @@ namespace :gitlab do desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' task configure: :environment do - if ActiveRecord::Base.connection.tables.any? + # Check if we have existing db tables + # The schema_migrations table will still exist if drop_tables was called + if ActiveRecord::Base.connection.tables.count > 1 Rake::Task['db:migrate'].invoke else Rake::Task['db:schema:load'].invoke diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index 44074397c05..900dbf7be24 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -10,15 +10,22 @@ namespace :gitlab do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end - desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' - task bump_test_version: :environment do - Dir.mktmpdir do |tmp_dir| - system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") - File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') - system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + desc 'GitLab | Bumps the Import/Export version in fixtures and project templates' + task bump_version: :environment do + archives = Dir['vendor/project_templates/*.tar.gz'] + archives.push('spec/features/projects/import_export/test_project_export.tar.gz') + + archives.each do |archive| + raise ArgumentError unless File.exist?(archive) + + Dir.mktmpdir do |tmp_dir| + system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null") + end end - puts "Updated to #{Gitlab::ImportExport.version}" + puts "Updated #{archives} to #{Gitlab::ImportExport.version}." end end end |