diff options
Diffstat (limited to 'lib')
26 files changed, 423 insertions, 436 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 40df1e79bc7..66b62d9ee2e 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -775,33 +775,28 @@ module API class Todo < Grape::Entity expose :id - expose :project, using: Entities::ProjectIdentity, if: -> (todo, _) { todo.project_id } - expose :group, using: 'API::Entities::NamespaceBasic', if: -> (todo, _) { todo.group_id } + expose :project, using: Entities::BasicProjectDetails expose :author, using: Entities::UserBasic expose :action_name expose :target_type expose :target do |todo, options| - todo_target_class(todo.target_type).represent(todo.target, options) + Entities.const_get(todo.target_type).represent(todo.target, options) end expose :target_url do |todo, options| target_type = todo.target_type.underscore - target_url = "#{todo.parent.class.to_s.underscore}_#{target_type}_url" + target_url = "namespace_project_#{target_type}_url" target_anchor = "note_#{todo.note_id}" if todo.note_id? Gitlab::Routing .url_helpers - .public_send(target_url, todo.parent, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend + .public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend end expose :body expose :state expose :created_at - - def todo_target_class(target_type) - ::API::Entities.const_get(target_type) - end end class NamespaceBasic < Grape::Entity diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 5c63ec028d9..fa828f43001 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -89,9 +89,10 @@ module API requires :environment_id, type: Integer, desc: 'The environment ID' end post ':id/environments/:environment_id/stop' do - authorize! :create_deployment, user_project + authorize! :read_environment, user_project environment = user_project.environments.find(params[:environment_id]) + authorize! :stop_environment, environment environment.stop_with_action!(current_user) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index f633dd88d06..797b04df059 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -150,12 +150,13 @@ module API end params do use :with_custom_attributes + optional :with_projects, type: Boolean, default: true, desc: 'Omit project details' end get ":id" do group = find_group!(params[:id]) options = { - with: Entities::GroupDetail, + with: params[:with_projects] ? Entities::GroupDetail : Entities::Group, current_user: current_user } diff --git a/lib/api/projects.rb b/lib/api/projects.rb index b83da00502d..8273abe48c9 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -260,7 +260,8 @@ module API :snippets_enabled, :tag_list, :visibility, - :wiki_enabled + :wiki_enabled, + :avatar ] optional :name, type: String, desc: 'The name of the project' optional :default_branch, type: String, desc: 'The default branch of the project' diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 2071c5a62c1..51242341dba 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -58,7 +58,7 @@ module API optional :access_level, type: String, values: Ci::Runner.access_levels.keys, desc: 'The access_level of the runner' optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' - at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level, :maximum_timeout end put ':id' do runner = get_runner(params.delete(:id)) diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index 1dd2855063d..dda6cd38dcd 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -21,7 +21,17 @@ module DeclarativePolicy cache = opts[:cache] || {} key = Cache.policy_key(user, subject) - cache[key] ||= class_for(subject).new(user, subject, opts) + cache[key] ||= + if Gitlab.rails5? + # to avoid deadlocks in multi-threaded environment when + # autoloading is enabled, we allow concurrent loads, + # https://gitlab.com/gitlab-org/gitlab-ce/issues/48263 + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + class_for(subject).new(user, subject, opts) + end + else + class_for(subject).new(user, subject, opts) + end end def class_for(subject) diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 87e377de4d3..b170145f013 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -7,12 +7,14 @@ module Gitlab module Access AccessDeniedError = Class.new(StandardError) - NO_ACCESS = 0 - GUEST = 10 - REPORTER = 20 - DEVELOPER = 30 - MASTER = 40 - OWNER = 50 + NO_ACCESS = 0 + GUEST = 10 + REPORTER = 20 + DEVELOPER = 30 + MAINTAINER = 40 + # @deprecated + MASTER = MAINTAINER + OWNER = 50 # Branch protection settings PROTECTION_NONE = 0 @@ -32,7 +34,7 @@ module Gitlab "Guest" => GUEST, "Reporter" => REPORTER, "Developer" => DEVELOPER, - "Maintainer" => MASTER + "Maintainer" => MAINTAINER } end @@ -44,10 +46,10 @@ module Gitlab def sym_options { - guest: GUEST, - reporter: REPORTER, - developer: DEVELOPER, - master: MASTER + guest: GUEST, + reporter: REPORTER, + developer: DEVELOPER, + maintainer: MAINTAINER } end diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb index 8fb2c334048..664ead1af44 100644 --- a/lib/gitlab/background_migration/delete_diff_files.rb +++ b/lib/gitlab/background_migration/delete_diff_files.rb @@ -4,41 +4,77 @@ module Gitlab module BackgroundMigration class DeleteDiffFiles - def perform(merge_request_diff_id) - merge_request_diff = MergeRequestDiff.find_by(id: merge_request_diff_id) + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' - return unless merge_request_diff - return unless should_delete_diff_files?(merge_request_diff) + belongs_to :merge_request + has_many :merge_request_diff_files + end - MergeRequestDiff.transaction do - merge_request_diff.update_column(:state, 'without_files') - - # explain (analyze, buffers) when deleting 453 diff files: - # - # Delete on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actual time=43.265..43.265 rows=0 loops=1) - # Buffers: shared hit=2043 read=259 dirtied=254 - # -> Index Scan using index_merge_request_diff_files_on_mr_diff_id_and_order on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actu - # al time=0.466..26.317 rows=453 loops=1) - # Index Cond: (merge_request_diff_id = 463448) - # Buffers: shared hit=17 read=84 - # Planning time: 0.107 ms - # Execution time: 43.287 ms - # - MergeRequestDiffFile.where(merge_request_diff_id: merge_request_diff.id).delete_all + class MergeRequestDiffFile < ActiveRecord::Base + self.table_name = 'merge_request_diff_files' + end + + DEAD_TUPLES_THRESHOLD = 50_000 + VACUUM_WAIT_TIME = 5.minutes + + def perform(ids) + @ids = ids + + # We should reschedule until deadtuples get in a desirable + # state (e.g. < 50_000). That may take more than one reschedule. + # + if should_wait_deadtuple_vacuum? + reschedule + return end + + prune_diff_files + end + + def should_wait_deadtuple_vacuum? + return false unless Gitlab::Database.postgresql? + + diff_files_dead_tuples_count >= DEAD_TUPLES_THRESHOLD end private - def should_delete_diff_files?(merge_request_diff) - return false if merge_request_diff.state == 'without_files' + def reschedule + BackgroundMigrationWorker.perform_in(VACUUM_WAIT_TIME, self.class.name.demodulize, [@ids]) + end + + def diffs_collection + MergeRequestDiff.where(id: @ids) + end + + def diff_files_dead_tuples_count + dead_tuple = + execute_statement("SELECT n_dead_tup FROM pg_stat_all_tables "\ + "WHERE relname = 'merge_request_diff_files'")[0] - merge_request = merge_request_diff.merge_request + dead_tuple&.fetch('n_dead_tup', 0).to_i + end + + def prune_diff_files + removed = 0 + updated = 0 - return false unless merge_request.state == 'merged' - return false if merge_request_diff.id == merge_request.latest_merge_request_diff_id + MergeRequestDiff.transaction do + updated = diffs_collection.update_all(state: 'without_files') + removed = MergeRequestDiffFile.where(merge_request_diff_id: @ids).delete_all + end + + log_info("Removed #{removed} merge_request_diff_files rows, "\ + "updated #{updated} merge_request_diffs rows") + end + + def execute_statement(sql) + ActiveRecord::Base.connection.execute(sql) + end - true + def log_info(message) + Rails.logger.info("BackgroundMigration::DeleteDiffFiles - #{message}") end end end diff --git a/lib/gitlab/background_migration/schedule_diff_files_deletion.rb b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb new file mode 100644 index 00000000000..609cf19187c --- /dev/null +++ b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class ScheduleDiffFilesDeletion + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + belongs_to :merge_request + + include EachBatch + end + + DIFF_BATCH_SIZE = 5_000 + INTERVAL = 5.minutes + MIGRATION = 'DeleteDiffFiles' + + def perform + diffs = MergeRequestDiff + .from("(#{diffs_collection.to_sql}) merge_request_diffs") + .where('merge_request_diffs.id != merge_request_diffs.latest_merge_request_diff_id') + .select(:id) + + diffs.each_batch(of: DIFF_BATCH_SIZE) do |relation, index| + ids = relation.pluck(:id) + + BackgroundMigrationWorker.perform_in(index * INTERVAL, MIGRATION, [ids]) + end + end + + private + + def diffs_collection + MergeRequestDiff + .joins(:merge_request) + .where("merge_requests.state = 'merged'") + .where('merge_requests.latest_merge_request_diff_id IS NOT NULL') + .where("merge_request_diffs.state NOT IN ('without_files', 'empty')") + .select('merge_requests.latest_merge_request_diff_id, merge_request_diffs.id') + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 0bbd60d8ffe..375d8bc1ff5 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -7,14 +7,15 @@ module Gitlab module Artifacts class Metadata ParserError = Class.new(StandardError) + InvalidStreamError = Class.new(StandardError) VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} - attr_reader :file, :path, :full_version + attr_reader :stream, :path, :full_version - def initialize(file, path, **opts) - @file, @path, @opts = file, path, opts + def initialize(stream, path, **opts) + @stream, @path, @opts = stream, path, opts @full_version = read_version end @@ -103,7 +104,17 @@ module Gitlab end def gzip(&block) - Zlib::GzipReader.open(@file, &block) + raise InvalidStreamError, "Invalid stream" unless @stream + + # restart gzip reading + @stream.seek(0) + + gz = Zlib::GzipReader.new(@stream) + yield(gz) + rescue Zlib::Error => e + raise InvalidStreamError, e.message + ensure + gz&.finish end end end diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb deleted file mode 100644 index 8788af57a67..00000000000 --- a/lib/gitlab/ci/trace/http_io.rb +++ /dev/null @@ -1,197 +0,0 @@ -## -# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) -# source: https://gitlab.com/snippets/1685610 -module Gitlab - module Ci - class Trace - class HttpIO - BUFFER_SIZE = 128.kilobytes - - InvalidURLError = Class.new(StandardError) - FailedToGetChunkError = Class.new(StandardError) - - attr_reader :uri, :size - attr_reader :tell - attr_reader :chunk, :chunk_range - - alias_method :pos, :tell - - def initialize(url, size) - raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) - - @uri = URI(url) - @size = size - @tell = 0 - end - - def close - # no-op - end - - def binmode - # no-op - end - - def binmode? - true - end - - def path - nil - end - - def url - @uri.to_s - end - - def seek(pos, where = IO::SEEK_SET) - new_pos = - case where - when IO::SEEK_END - size + pos - when IO::SEEK_SET - pos - when IO::SEEK_CUR - tell + pos - else - -1 - end - - raise 'new position is outside of file' if new_pos < 0 || new_pos > size - - @tell = new_pos - end - - def eof? - tell == size - end - - def each_line - until eof? - line = readline - break if line.nil? - - yield(line) - end - end - - def read(length = nil, outbuf = "") - out = "" - - length ||= size - tell - - until length <= 0 || eof? - data = get_chunk - break if data.empty? - - chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min - chunk_data = data.byteslice(0, chunk_bytes) - - out << chunk_data - @tell += chunk_data.bytesize - length -= chunk_data.bytesize - end - - # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality - if outbuf - outbuf.slice!(0, outbuf.bytesize) - outbuf << out - end - - out - end - - def readline - out = "" - - until eof? - data = get_chunk - new_line = data.index("\n") - - if !new_line.nil? - out << data[0..new_line] - @tell += new_line + 1 - break - else - out << data - @tell += data.bytesize - end - end - - out - end - - def write(data) - raise NotImplementedError - end - - def truncate(offset) - raise NotImplementedError - end - - def flush - raise NotImplementedError - end - - def present? - true - end - - private - - ## - # The below methods are not implemented in IO class - # - def in_range? - @chunk_range&.include?(tell) - end - - def get_chunk - unless in_range? - response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http| - http.request(request) - end - - raise FailedToGetChunkError unless response.code == '200' || response.code == '206' - - @chunk = response.body.force_encoding(Encoding::BINARY) - @chunk_range = response.content_range - - ## - # Note: If provider does not return content_range, then we set it as we requested - # Provider: minio - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # Provider: AWS - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # Provider: GCS - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 - @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize)) - end - - @chunk[chunk_offset..BUFFER_SIZE] - end - - def request - Net::HTTP::Get.new(uri).tap do |request| - request.set_range(chunk_start, BUFFER_SIZE) - end - end - - def chunk_offset - tell % BUFFER_SIZE - end - - def chunk_start - (tell / BUFFER_SIZE) * BUFFER_SIZE - end - - def chunk_end - [chunk_start + BUFFER_SIZE, size].min - end - end - end - end -end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 8c72d00c1f3..ee604e66154 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -138,15 +138,23 @@ module Gitlab def ee_branch_presence_check! ee_remotes.keys.each do |remote| - [ce_branch, ee_branch_prefix, ee_branch_suffix].each do |branch| - _, status = step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) + output, _ = step( + "Searching #{remote}", + %W[git ls-remote #{remote} *#{minimal_ee_branch_name}*]) - if status.zero? - @ee_remote_with_branch = remote - @ee_branch_found = branch - return true - end - end + branches = + output.scan(%r{(?<=refs/heads/|refs/tags/).+}).sort_by(&:size) + + next if branches.empty? + + branch = branches.first + + step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) + + @ee_remote_with_branch = remote + @ee_branch_found = branch + + return true end puts @@ -271,6 +279,10 @@ module Gitlab @ee_patch_full_path ||= patches_dir.join(ee_patch_name) end + def minimal_ee_branch_name + @minimal_ee_branch_name ||= ce_branch.sub(/(\Ace\-|\-ce\z)/, '') + end + def patch_name_from_branch(branch_name) branch_name.parameterize << '.patch' end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 96fa94d5790..71857bd2d87 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -61,17 +61,8 @@ module Gitlab # Keep in mind that this method may allocate a lot of memory. It is up # to the caller to limit the number of blobs and blob_size_limit. # - # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) - Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| - if is_enabled - repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a - else - blob_references.map do |sha, path| - find(repository, sha, path, limit: blob_size_limit) - end - end - end + repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a end # Returns an array of Blob instances just with the metadata, that means @@ -84,16 +75,8 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled| - if is_enabled - repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) - else - blob_ids.lazy - .select { |sha| possible_lfs_blob?(repository, sha) } - .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } - .select(&:lfs_pointer?) - .force - end + repository.wrapped_gitaly_errors do + repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) end end @@ -104,72 +87,6 @@ module Gitlab def size_could_be_lfs?(size) size.between?(LFS_POINTER_MIN_SIZE, LFS_POINTER_MAX_SIZE) end - - private - - # Recursive search of blob id by path - # - # Ex. - # blog/ # oid: 1a - # app/ # oid: 2a - # models/ # oid: 3a - # file.rb # oid: 4a - # - # - # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' - # - def find_entry_by_path(repository, root_id, *path_parts) - root_tree = repository.lookup(root_id) - - entry = root_tree.find do |entry| - entry[:name] == path_parts[0] - end - - return nil unless entry - - if path_parts.size > 1 - return nil unless entry[:type] == :tree - - path_parts.shift - find_entry_by_path(repository, entry[:oid], *path_parts) - else - [:blob, :commit].include?(entry[:type]) ? entry : nil - end - end - - def submodule_blob(blob_entry, path, sha) - new( - id: blob_entry[:oid], - name: blob_entry[:name], - size: 0, - data: '', - path: path, - commit_id: sha - ) - end - - def rugged_raw(repository, sha, limit:) - blob = repository.lookup(sha) - - return unless blob.is_a?(Rugged::Blob) - - new( - id: blob.oid, - size: blob.size, - data: blob.content(limit), - binary: blob.binary? - ) - end - - # Efficient lookup to determine if object size - # and type make it a possible LFS blob without loading - # blob content into memory with repository.lookup(sha) - def possible_lfs_blob?(repository, sha) - object_header = repository.rugged.read_header(sha) - - object_header[:type] == :blob && - size_could_be_lfs?(object_header[:len]) - end end def initialize(options) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 76404366e8e..a1a050647b9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -627,7 +627,7 @@ module Gitlab def update_branch(branch_name, user:, newrev:, oldrev:) gitaly_migrate(:operation_user_update_branch) do |is_enabled| if is_enabled - gitaly_operations_client.user_update_branch(branch_name, user, newrev, oldrev) + gitaly_operation_client.user_update_branch(branch_name, user, newrev, oldrev) else OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) end @@ -904,12 +904,8 @@ module Gitlab end def fetch_source_branch!(source_repository, source_branch, local_ref) - Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled| - if is_enabled - gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) - else - rugged_fetch_source_branch(source_repository, source_branch, local_ref) - end + wrapped_gitaly_errors do + gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) end end @@ -1064,12 +1060,8 @@ module Gitlab end def bundle_to_disk(save_path) - gitaly_migrate(:bundle_to_disk) do |is_enabled| - if is_enabled - gitaly_repository_client.create_bundle(save_path) - else - run_git!(%W(bundle create #{save_path} --all)) - end + wrapped_gitaly_errors do + gitaly_repository_client.create_bundle(save_path) end true diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index db7c29be94b..35808149b90 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,6 +2,8 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess + include Gitlab::Utils::StrongMemoize + UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) @@ -26,7 +28,7 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @@ -40,6 +42,8 @@ module Gitlab end def check(cmd, changes) + @changes = changes + check_protocol! check_valid_actor! check_active_user! @@ -58,7 +62,7 @@ module Gitlab when *DOWNLOAD_COMMANDS check_download_access! when *PUSH_COMMANDS - check_push_access!(changes) + check_push_access! end true @@ -218,7 +222,7 @@ module Gitlab end end - def check_push_access!(changes) + def check_push_access! if project.repository_read_only? raise UnauthorizedError, ERROR_MESSAGES[:read_only] end @@ -235,17 +239,15 @@ module Gitlab return if changes.blank? # Allow access this is needed for EE. - check_change_access!(changes) + check_change_access! end - def check_change_access!(changes) + def check_change_access! # If there are worktrees with a HEAD pointing to a non-existent object, # calls to `git rev-list --all` will fail in git 2.15+. This should also # clear stale lock files. project.repository.clean_stale_repository_files - changes_list = Gitlab::ChangesList.new(changes) - # Iterate over all changes to find if user allowed all of them to be applied changes_list.each.with_index do |change, index| first_change = index == 0 @@ -321,6 +323,10 @@ module Gitlab protected + def changes_list + @changes_list ||= Gitlab::ChangesList.new(changes) + end + def user return @user if defined?(@user) diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index b1a01b185e6..aa7e03301f5 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -25,10 +25,12 @@ module Gitlab def conflicts? list_conflict_files.any? - rescue GRPC::FailedPrecondition - # The server raises this exception when it encounters ConflictSideMissing, which - # means a conflict exists but its `theirs` or `ours` data is nil due to a non-existent - # file in one of the trees. + rescue GRPC::FailedPrecondition, GRPC::Unknown + # The server raises FailedPrecondition when it encounters + # ConflictSideMissing, which means a conflict exists but its `theirs` or + # `ours` data is nil due to a non-existent file in one of the trees. + # + # GRPC::Unknown comes from Rugged::ReferenceError and Rugged::OdbError. true end diff --git a/lib/gitlab/http_io.rb b/lib/gitlab/http_io.rb new file mode 100644 index 00000000000..ce24817db54 --- /dev/null +++ b/lib/gitlab/http_io.rb @@ -0,0 +1,193 @@ +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + class HttpIO + BUFFER_SIZE = 128.kilobytes + + InvalidURLError = Class.new(StandardError) + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :uri, :size + attr_reader :tell + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(url, size) + raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) + + @uri = URI(url) + @size = size + @tell = 0 + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def path + nil + end + + def url + @uri.to_s + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil, outbuf = "") + out = "" + + length ||= size - tell + + until length <= 0 || eof? + data = get_chunk + break if data.empty? + + chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min + chunk_data = data.byteslice(0, chunk_bytes) + + out << chunk_data + @tell += chunk_data.bytesize + length -= chunk_data.bytesize + end + + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality + if outbuf + outbuf.slice!(0, outbuf.bytesize) + outbuf << out + end + + out + end + + def readline + out = "" + + until eof? + data = get_chunk + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out + end + + def write(data) + raise NotImplementedError + end + + def truncate(offset) + raise NotImplementedError + end + + def flush + raise NotImplementedError + end + + def present? + true + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def get_chunk + unless in_range? + response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http| + http.request(request) + end + + raise FailedToGetChunkError unless response.code == '200' || response.code == '206' + + @chunk = response.body.force_encoding(Encoding::BINARY) + @chunk_range = response.content_range + + ## + # Note: If provider does not return content_range, then we set it as we requested + # Provider: minio + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: AWS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: GCS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 + @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize)) + end + + @chunk[chunk_offset..BUFFER_SIZE] + end + + def request + Net::HTTP::Get.new(uri).tap do |request| + request.set_range(chunk_start, BUFFER_SIZE) + end + end + + def chunk_offset + tell % BUFFER_SIZE + end + + def chunk_start + (tell / BUFFER_SIZE) * BUFFER_SIZE + end + + def chunk_end + [chunk_start + BUFFER_SIZE, size].min + end + end +end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 8b8e48aac76..ac827cbe1ca 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -47,7 +47,7 @@ module Gitlab def ensure_default_member! @project.project_members.destroy_all - ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) end def add_team_member(member, existing_user = nil) diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index da43bd0af4b..15c5ece2350 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -1,6 +1,10 @@ module Gitlab # Helper methods to do with Kubernetes network services & resources module Kubernetes + def self.build_header_hash + Hash.new { |h, k| h[k] = [] } + end + # This is the comand that is run to start a terminal session. Kubernetes # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` EXEC_COMMAND = URI.encode_www_form( @@ -37,13 +41,14 @@ module Gitlab selectors: { pod: pod_name, container: container["name"] }, url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], - headers: Hash.new { |h, k| h[k] = [] }, + headers: ::Gitlab::Kubernetes.build_header_hash, created_at: created_at } end end def add_terminal_auth(terminal, token:, max_session_time:, ca_pem: nil) + terminal[:headers] ||= ::Gitlab::Kubernetes.build_header_hash terminal[:headers]['Authorization'] << "Bearer #{token}" terminal[:max_session_time] = max_session_time terminal[:ca_pem] = ca_pem if ca_pem.present? diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb index 15b8beacf60..e3da1634fa5 100644 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -24,7 +24,7 @@ module Gitlab user.projects.select_for_project_authorization, # The personal projects of the user. - user.personal_projects.select_as_master_for_project_authorization, + user.personal_projects.select_as_maintainer_for_project_authorization, # Projects that belong directly to any of the groups the user has # access to. diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb index ad87540e6c2..7d0c00c7f36 100644 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -15,7 +15,7 @@ module Gitlab user.projects.select_for_project_authorization, # Personal projects - user.personal_projects.select_as_master_for_project_authorization, + user.personal_projects.select_as_maintainer_for_project_authorization, # Projects of groups the user is a member of user.groups_projects.select_for_project_authorization, diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index e222541992a..a17cd27e82d 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -92,21 +92,13 @@ module Gitlab # Ex. # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) if url.start_with?('.', '/') raise Error.new("don't use disk paths with import_repository: #{url.inspect}") end relative_path = "#{name}.git" - cmd = gitaly_migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - GitalyGitlabProjects.new(storage, relative_path) - else - # The timeout ensures the subprocess won't hang forever - gitlab_projects(storage, relative_path) - end - end + cmd = GitalyGitlabProjects.new(storage, relative_path) success = cmd.import_project(url, git_timeout) raise Error, cmd.output unless success @@ -126,12 +118,8 @@ module Gitlab # fetch_remote(my_repo, "upstream") # def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - gitaly_migrate(:fetch_remote) do |is_enabled| - if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) - else - local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) - end + wrapped_gitaly_errors do + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) end end @@ -389,28 +377,6 @@ module Gitlab ) end - def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - vars = { force: forced, tags: !no_tags, prune: prune } - - if ssh_auth&.ssh_import? - if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? - vars[:ssh_key] = ssh_auth.ssh_private_key - end - - if ssh_auth.ssh_known_hosts.present? - vars[:known_hosts] = ssh_auth.ssh_known_hosts - end - end - - cmd = gitlab_projects(storage_name, repository_relative_path) - - success = cmd.fetch_remote(remote, git_timeout, vars) - - raise Error, cmd.output unless success - - success - end - def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -440,10 +406,6 @@ module Gitlab Gitlab.config.gitlab_shell.git_timeout end - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - wrapped_gitaly_errors { Gitlab::GitalyClient.migrate(method, status: status, &block) } - end - def wrapped_gitaly_errors yield rescue GRPC::NotFound, GRPC::BadStatus => e diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 59331c827af..de8b6ec69ce 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -58,7 +58,7 @@ module Gitlab if raw_credentials.present? url.sub!("#{raw_credentials}@", '') - user, password = raw_credentials.split(':') + user, _, password = raw_credentials.partition(':') @credentials ||= { user: user.presence, password: password.presence } end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 55c899912f9..a9629a92a50 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -98,16 +98,12 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) - { - 'GitalyServer' => gitaly_server_hash(repository), - 'RawPatchRequest' => Gitaly::RawPatchRequest.new( - gitaly_diff_or_patch_hash(repository, diff_refs) - ).to_json - } - else - workhorse_diff_or_patch_hash(repository, diff_refs) - end + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawPatchRequest' => Gitaly::RawPatchRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } [ SEND_DATA_HEADER, @@ -220,14 +216,6 @@ module Gitlab } end - def workhorse_diff_or_patch_hash(repository, diff_refs) - { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } - end - def gitaly_diff_or_patch_hash(repository, diff_refs) { repository: repository.gitaly_repository, diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 6df7fe81437..f431352b61e 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -20,16 +20,22 @@ namespace :gettext do end task :regenerate do + pot_file = 'locale/gitlab.pot' # Remove all translated files, this speeds up finding FileUtils.rm Dir['locale/**/gitlab.*'] # remove the `pot` file to ensure it's completely regenerated - FileUtils.rm_f 'locale/gitlab.pot' + FileUtils.rm_f pot_file Rake::Task['gettext:find'].invoke # leave only the required changes. `git checkout -- locale/*/gitlab.po` + # Remove timestamps from the pot file + pot_content = File.read pot_file + pot_content.gsub!(/^"POT?\-(?:Creation|Revision)\-Date\:.*\n/, '') + File.write pot_file, pot_content + puts <<~MSG All done. Please commit the changes to `locale/gitlab.pot`. diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index 83dd870fa31..26cbf0740b6 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -1,6 +1,6 @@ namespace :gitlab do namespace :import do - desc "GitLab | Add all users to all projects (admin users are added as masters)" + desc "GitLab | Add all users to all projects (admin users are added as maintainers)" task all_users_to_all_projects: :environment do |t, args| user_ids = User.where(admin: false).pluck(:id) admin_ids = User.where(admin: true).pluck(:id) @@ -10,7 +10,7 @@ namespace :gitlab do ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER) puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects" - ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MASTER) + ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MAINTAINER) end desc "GitLab | Add a specific user to all projects (as a developer)" |