diff options
| author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 20:36:55 +0100 |
|---|---|---|
| committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 20:36:55 +0100 |
| commit | 965dc28691e2d70b7040e28d90ccbc3721a9e416 (patch) | |
| tree | 84258f35b72f2e7ce6a7198db66032df4ad5aadb /lib | |
| parent | e3fafa7632e038927085cf8c8228c93be44b36bd (diff) | |
| parent | 7fabc892f251740dbd9a4755baede662e6854870 (diff) | |
| download | gitlab-ce-965dc28691e2d70b7040e28d90ccbc3721a9e416.tar.gz | |
Merge commit '7fabc892f251740dbd9a4755baede662e6854870' into object-storage-ee-to-ce-backport
Diffstat (limited to 'lib')
288 files changed, 6093 insertions, 2151 deletions
diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index 4750a2c373a..db63c5038ae 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -6,12 +6,34 @@ module AfterCommitQueue after_rollback :_clear_after_commit_queue end - def run_after_commit(method = nil, &block) - _after_commit_queue << proc { self.send(method) } if method # rubocop:disable GitlabSecurity/PublicSend + def run_after_commit(&block) _after_commit_queue << block if block + + true + end + + def run_after_commit_or_now(&block) + if AfterCommitQueue.inside_transaction? + run_after_commit(&block) + else + instance_eval(&block) + end + true end + def self.open_transactions_baseline + if ::Rails.env.test? + return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) } + end + + 0 + end + + def self.inside_transaction? + ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + end + protected def _run_after_commit_queue diff --git a/lib/api/api.rb b/lib/api/api.rb index c37e596eb9d..8094597d238 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -61,7 +61,10 @@ module API mount ::API::V3::Variables end - before { header['X-Frame-Options'] = 'SAMEORIGIN' } + before do + header['X-Frame-Options'] = 'SAMEORIGIN' + header['X-Content-Type-Options'] = 'nosniff' + end # The locale is set to the current user's locale when `current_user` is loaded after { Gitlab::I18n.use_default_locale } diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index b9c7d443f6c..9aeebc34525 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,9 +6,6 @@ module API module APIGuard extend ActiveSupport::Concern - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN".freeze - PRIVATE_TOKEN_PARAM = :private_token - included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| @@ -42,6 +39,8 @@ module API # Helper Methods for Grape Endpoint module HelperMethods + include Gitlab::Auth::UserAuthFinders + def find_current_user! user = find_user_from_access_token || find_user_from_warden return unless user @@ -51,76 +50,8 @@ module API user end - def access_token - return @access_token if defined?(@access_token) - - @access_token = find_oauth_access_token || find_personal_access_token - end - - def validate_access_token!(scopes: []) - return unless access_token - - case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) - when AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when AccessTokenValidationService::EXPIRED - raise ExpiredError - when AccessTokenValidationService::REVOKED - raise RevokedError - end - end - private - def find_user_from_access_token - return unless access_token - - validate_access_token! - - access_token.user || raise(UnauthorizedError) - end - - # Check the Rails session for valid authentication details - def find_user_from_warden - warden.try(:authenticate) if verified_request? - end - - def warden - env['warden'] - end - - # Check if the request is GET/HEAD, or if CSRF token is valid. - def verified_request? - Gitlab::RequestForgeryProtection.verified?(env) - end - - def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) - return unless token - - # Expiration, revocation and scopes are verified in `find_user_by_access_token` - access_token = OauthAccessToken.by_token(token) - raise UnauthorizedError unless access_token - - access_token.revoke_previous_refresh_token! - access_token - end - - def find_personal_access_token - token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - return unless token.present? - - # Expiration, revocation and scopes are verified in `find_user_by_access_token` - access_token = PersonalAccessToken.find_by(token: token) - raise UnauthorizedError unless access_token - - access_token - end - - def doorkeeper_request - @doorkeeper_request ||= ActionDispatch::Request.new(env) - end - # An array of scopes that were registered (using `allow_access_with_scope`) # for the current endpoint class. It also returns scopes registered on # `API::API`, since these are meant to apply to all API routes. @@ -143,8 +74,11 @@ module API private def install_error_responders(base) - error_classes = [MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + error_classes = [Gitlab::Auth::MissingTokenError, + Gitlab::Auth::TokenNotFoundError, + Gitlab::Auth::ExpiredError, + Gitlab::Auth::RevokedError, + Gitlab::Auth::InsufficientScopeError] base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend end @@ -153,25 +87,25 @@ module API proc do |e| response = case e - when MissingTokenError + when Gitlab::Auth::MissingTokenError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new - when TokenNotFoundError + when Gitlab::Auth::TokenNotFoundError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Bad Access Token.") - when ExpiredError + when Gitlab::Auth::ExpiredError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Token is expired. You can either do re-authorization or token refresh.") - when RevokedError + when Gitlab::Auth::RevokedError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Token was revoked. You have to re-authorize from the user.") - when InsufficientScopeError + when Gitlab::Auth::InsufficientScopeError # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) # does not include WWW-Authenticate header, which breaks the standard. Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( @@ -184,22 +118,5 @@ module API end end end - - # - # Exceptions - # - - MissingTokenError = Class.new(StandardError) - TokenNotFoundError = Class.new(StandardError) - ExpiredError = Class.new(StandardError) - RevokedError = Class.new(StandardError) - UnauthorizedError = Class.new(StandardError) - - class InsufficientScopeError < StandardError - attr_reader :scopes - def initialize(scopes) - @scopes = scopes.map { |s| s.try(:name) || s } - end - end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index cdef1b546a9..0791a110c39 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -81,9 +81,9 @@ module API service_args = [user_project, current_user, protected_branch_params] protected_branch = if protected_branch - ::ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + ::ProtectedBranches::LegacyApiUpdateService.new(*service_args).execute(protected_branch) else - ::ProtectedBranches::ApiCreateService.new(*service_args).execute + ::ProtectedBranches::LegacyApiCreateService.new(*service_args).execute end if protected_branch.valid? diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb index 118883f5ea5..598c76f6168 100644 --- a/lib/api/circuit_breakers.rb +++ b/lib/api/circuit_breakers.rb @@ -41,7 +41,7 @@ module API detail 'This feature was introduced in GitLab 9.5' end delete do - Gitlab::Git::Storage::CircuitBreaker.reset_all! + Gitlab::Git::Storage::FailureInfo.reset_all! end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 2685dc27252..38e05074353 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -117,7 +117,7 @@ module API commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - notes = user_project.notes.where(commit_id: commit.id).order(:created_at) + notes = commit.notes.order(:created_at) present paginate(notes), with: Entities::CommitNote end @@ -180,10 +180,12 @@ module API if params[:path] commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index a382db92e8d..d96e7f2770f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -16,10 +16,13 @@ module API class UserBasic < UserSafe expose :state + expose :avatar_url do |user, options| user.avatar_url(only_path: false) end + expose :avatar_path, if: ->(user, options) { options.fetch(:only_path, false) && user.avatar_path } + expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) end @@ -80,16 +83,37 @@ module API expose :group_access, as: :group_access_level end - class BasicProjectDetails < Grape::Entity - expose :id, :description, :default_branch, :tag_list - expose :ssh_url_to_repo, :http_url_to_repo, :web_url + class ProjectIdentity < Grape::Entity + expose :id, :description expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :created_at + end + + class BasicProjectDetails < ProjectIdentity + include ::API::ProjectsRelationBuilder + + expose :default_branch + # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 + expose :tag_list do |project| + # project.tags.order(:name).pluck(:name) is the most suitable option + # to avoid loading all the ActiveRecord objects but, if we use it here + # it override the preloaded associations and makes a query + # (fixed in https://github.com/rails/rails/pull/25976). + project.tags.map(&:name).sort + end + expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :avatar_url do |project, options| project.avatar_url(only_path: false) end expose :star_count, :forks_count - expose :created_at, :last_activity_at + expose :last_activity_at + + def self.preload_relation(projects_relation, options = {}) + projects_relation.preload(:project_feature, :route) + .preload(namespace: [:route, :owner], + tags: :taggings) + end end class Project < BasicProjectDetails @@ -141,7 +165,7 @@ module API expose :shared_runners_enabled expose :lfs_enabled?, as: :lfs_enabled expose :creator_id - expose :namespace, using: 'API::Entities::Namespace' + expose :namespace, using: 'API::Entities::NamespaceBasic' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -151,7 +175,7 @@ module API expose :public_builds, as: :public_jobs expose :ci_config_path expose :shared_with_groups do |project, options| - SharedGroup.represent(project.project_group_links.all, options) + SharedGroup.represent(project.project_group_links, options) end expose :only_allow_merge_if_pipeline_succeeds expose :request_access_enabled @@ -159,6 +183,18 @@ module API expose :printing_merge_request_link_enabled expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + + def self.preload_relation(projects_relation, options = {}) + super(projects_relation).preload(:group) + .preload(project_group_links: :group, + fork_network: :root_project, + forked_project_link: :forked_from_project, + forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) + end + + def self.forks_counting_projects(projects_relation) + projects_relation + projects_relation.map(&:forked_from_project).compact + end end class ProjectStatistics < Grape::Entity @@ -242,7 +278,11 @@ module API end expose :merged do |repo_branch, options| - options[:project].repository.merged_to_root_ref?(repo_branch, options[:merged_branch_names]) + if options[:merged_branch_names] + options[:merged_branch_names].include?(repo_branch.name) + else + options[:project].repository.merged_to_root_ref?(repo_branch) + end end expose :protected do |repo_branch, options| @@ -609,9 +649,11 @@ module API expose :created_at end - class Namespace < Grape::Entity + class NamespaceBasic < Grape::Entity expose :id, :name, :path, :kind, :full_path, :parent_id + end + class Namespace < NamespaceBasic expose :members_count_with_descendants, if: -> (namespace, opts) { expose_members_count_with_descendants?(namespace, opts) } do |namespace, _| namespace.users_with_descendants.count end @@ -671,7 +713,7 @@ module API if options.key?(:project_members) (options[:project_members] || []).find { |member| member.source_id == project.id } else - project.project_members.find_by(user_id: options[:current_user].id) + project.project_member(options[:current_user]) end end @@ -680,11 +722,25 @@ module API if options.key?(:group_members) (options[:group_members] || []).find { |member| member.source_id == project.namespace_id } else - project.group.group_members.find_by(user_id: options[:current_user].id) + project.group.group_member(options[:current_user]) end end end end + + def self.preload_relation(projects_relation, options = {}) + relation = super(projects_relation, options) + + unless options.key?(:group_members) + relation = relation.preload(group: [group_members: [:source, user: [notification_settings: :source]]]) + end + + unless options.key?(:project_members) + relation = relation.preload(project_members: [:source, user: [notification_settings: :source]]) + end + + relation + end end class LabelBasic < Grape::Entity @@ -763,7 +819,10 @@ module API expose(:default_project_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_project_visibility) } expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) } expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) } - expose :password_authentication_enabled, as: :signin_enabled + + # support legacy names, can be removed in v5 + expose :password_authentication_enabled_for_web, as: :password_authentication_enabled + expose :password_authentication_enabled_for_web, as: :signin_enabled end class Release < Grape::Entity @@ -820,17 +879,24 @@ module API expose :id, :sha, :ref, :status end - class Job < Grape::Entity + class JobBasic < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at expose :duration expose :user, with: User - expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :commit, with: Commit - expose :runner, with: Runner expose :pipeline, with: PipelineBasic end + class Job < JobBasic + expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } + expose :runner, with: Runner + end + + class JobBasicWithProject < JobBasic + expose :project, with: ProjectIdentity + end + class Trigger < Grape::Entity expose :id expose :token, :description @@ -987,13 +1053,9 @@ module API expose :type, :url, :username, :password end - class ArtifactFile < Grape::Entity - expose :filename, :size - end - class Dependency < Grape::Entity expose :id, :name, :token - expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? } + expose :artifacts_file, using: JobArtifactFile, if: ->(job, _) { job.artifacts? } end class Response < Grape::Entity @@ -1042,6 +1104,11 @@ module API expose :value end + class PagesDomainCertificateExpiration < Grape::Entity + expose :expired?, as: :expired + expose :expiration + end + class PagesDomainCertificate < Grape::Entity expose :subject expose :expired?, as: :expired @@ -1049,12 +1116,23 @@ module API expose :certificate_text end + class PagesDomainBasic < Grape::Entity + expose :domain + expose :url + expose :certificate, + as: :certificate_expiration, + if: ->(pages_domain, _) { pages_domain.certificate? }, + using: PagesDomainCertificateExpiration do |pages_domain| + pages_domain + end + end + class PagesDomain < Grape::Entity expose :domain expose :url expose :certificate, - if: ->(pages_domain, _) { pages_domain.certificate? }, - using: PagesDomainCertificate do |pages_domain| + if: ->(pages_domain, _) { pages_domain.certificate? }, + using: PagesDomainCertificate do |pages_domain| pages_domain end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 340a7cecf09..b81f07a1770 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -25,24 +25,7 @@ module API optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end - def present_groups(groups, options = {}) - options = options.reverse_merge( - with: Entities::Group, - current_user: current_user - ) - - groups = groups.with_statistics if options[:statistics] - present paginate(groups), options - end - end - - resource :groups do - include CustomAttributesEndpoints - - desc 'Get a groups list' do - success Entities::Group - end - params do + params :group_list_params do use :statistics_params optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' optional :all_available, type: Boolean, desc: 'Show all group that you have access to' @@ -52,19 +35,54 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end - get do + + def find_groups(params) find_params = { all_available: params[:all_available], - owned: params[:owned], - custom_attributes: params[:custom_attributes] + custom_attributes: params[:custom_attributes], + owned: params[:owned] } + find_params[:parent] = find_group!(params[:id]) if params[:id] groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present_groups groups, statistics: params[:statistics] && current_user.admin? + groups + end + + def find_group_projects(params) + group = find_group!(params[:id]) + projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) + paginate(projects) + end + + def present_groups(params, groups) + options = { + with: Entities::Group, + current_user: current_user, + statistics: params[:statistics] && current_user.admin? + } + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end + end + + resource :groups do + include CustomAttributesEndpoints + + desc 'Get a groups list' do + success Entities::Group + end + params do + use :group_list_params + end + get do + groups = find_groups(params) + present_groups params, groups end desc 'Create a group. Available only for users who can create groups.' do @@ -159,11 +177,21 @@ module API use :pagination end get ":id/projects" do - group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute - projects = reorder_projects(projects) + projects = find_group_projects(params) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project - present paginate(projects), with: entity, current_user: current_user + + present entity.prepare_relation(projects), with: entity, current_user: current_user + end + + desc 'Get a list of subgroups in this group.' do + success Entities::Group + end + params do + use :group_list_params + end + get ":id/subgroups" do + groups = find_groups(params) + present_groups params, groups end desc 'Transfer a project to the group namespace. Available only for admin.' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 6e58022a265..72ada181dd2 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -50,6 +50,10 @@ module API initial_current_user != current_user end + def user_namespace + @user_namespace ||= find_namespace!(params[:id]) + end + def user_group @group ||= find_group!(params[:id]) end @@ -112,6 +116,24 @@ module API end end + def find_namespace(id) + if id.to_s =~ /^\d+$/ + Namespace.find_by(id: id) + else + Namespace.find_by_full_path(id) + end + end + + def find_namespace!(id) + namespace = find_namespace(id) + + if can?(current_user, :read_namespace, namespace) + namespace + else + not_found!('Namespace') + end + end + def find_project_label(id) label = available_labels.find_by_id(id) || available_labels.find_by_title(id) label || not_found!('Label') @@ -155,6 +177,11 @@ module API end end + def authenticated_with_full_private_access! + authenticate! + forbidden! unless current_user.full_private_access? + end + def authenticated_as_admin! authenticate! forbidden! unless current_user.admin? @@ -190,6 +217,10 @@ module API not_found! unless user_project.pages_available? end + def require_pages_config_enabled! + not_found! unless Gitlab.config.pages.enabled + end + def can?(object, action, subject = :global) Ability.allowed?(object, action, subject) end @@ -389,7 +420,7 @@ module API begin @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user! } - rescue APIGuard::UnauthorizedError + rescue Gitlab::Auth::UnauthorizedError unauthorized! end end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index 0a8f3073a50..dd4f6c41131 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -4,6 +4,7 @@ module API class Absence < Grape::Validations::Base def validate_param!(attr_name, params) return if params.respond_to?(:key?) && !params.key?(attr_name) + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 4c0db4d42b1..d6dea4c30e3 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -2,8 +2,8 @@ module API module Helpers module InternalHelpers SSH_GITALY_FEATURES = { - 'git-receive-pack' => :ssh_receive_pack, - 'git-upload-pack' => :ssh_upload_pack + 'git-receive-pack' => [:ssh_receive_pack, Gitlab::GitalyClient::MigrationStatus::OPT_IN], + 'git-upload-pack' => [:ssh_upload_pack, Gitlab::GitalyClient::MigrationStatus::OPT_OUT] }.freeze def wiki? @@ -36,6 +36,18 @@ module API {} end + def fix_git_env_repository_paths(env, repository_path) + if obj_dir_relative = env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence + env['GIT_OBJECT_DIRECTORY'] = File.join(repository_path, obj_dir_relative) + end + + if alt_obj_dirs_relative = env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE'].presence + env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_obj_dirs_relative.map { |dir| File.join(repository_path, dir) } + end + + env + end + def log_user_activity(actor) commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS @@ -90,8 +102,8 @@ module API # Return the Gitaly Address if it is enabled def gitaly_payload(action) - feature = SSH_GITALY_FEATURES[action] - return unless feature && Gitlab::GitalyClient.feature_enabled?(feature) + feature, status = SSH_GITALY_FEATURES[action] + return unless feature && Gitlab::GitalyClient.feature_enabled?(feature, status: status) { repository: repository.gitaly_repository, diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 95108292aac..bb70370ba77 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -2,6 +2,8 @@ module API module Helpers module Pagination def paginate(relation) + relation = add_default_order(relation) + relation.page(params[:page]).per(params[:per_page]).tap do |data| add_pagination_headers(data) end @@ -45,6 +47,14 @@ module API # Ensure there is in total at least 1 page [paginated_data.total_pages, 1].max end + + def add_default_order(relation) + if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? + relation = relation.order(:id) + end + + relation + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 282af32ca94..2cae53dba53 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -14,6 +14,7 @@ module API def get_runner_version_from_params return unless params['info'].present? + attributes_for_keys(%w(name version revision platform architecture), params['info']) end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 6e78ac2c903..451121a4cea 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -19,7 +19,9 @@ module API status 200 # Stores some Git-specific env thread-safely - Gitlab::Git::Env.set(parse_env) + env = parse_env + env = fix_git_env_repository_paths(env, repository_path) if project + Gitlab::Git::Env.set(env) actor = if params[:key_id] diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0df41dcc903..e60e00d7956 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -68,7 +68,7 @@ module API desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' end get do - issues = find_issues + issues = paginate(find_issues) options = { with: Entities::IssueBasic, @@ -76,7 +76,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end end @@ -95,7 +95,7 @@ module API get ":id/issues" do group = find_group!(params[:id]) - issues = find_issues(group_id: group.id) + issues = paginate(find_issues(group_id: group.id)) options = { with: Entities::IssueBasic, @@ -103,7 +103,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end end @@ -124,7 +124,7 @@ module API get ":id/issues" do project = find_project!(params[:id]) - issues = find_issues(project_id: project.id) + issues = paginate(find_issues(project_id: project.id)) options = { with: Entities::IssueBasic, @@ -133,7 +133,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end desc 'Get a single project issue' do @@ -255,7 +255,9 @@ module API authorize!(:destroy_issue, issue) - destroy_conditionally!(issue) + destroy_conditionally!(issue) do |issue| + Issuable::DestroyService.new(user_project, current_user).execute(issue) + end end desc 'List merge requests closing issue' do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 3c1c412ba42..a116ab3c9bd 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -136,7 +136,7 @@ module API authorize_update_builds! build = find_build!(params[:job_id]) - authorize!(:update_build, build) + authorize!(:erase_build, build) return forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 726f09e3669..d34886fca2e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -21,7 +21,7 @@ module API return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels, :timelogs) + .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) end params :merge_requests_params do @@ -167,7 +167,9 @@ module API authorize!(:destroy_merge_request, merge_request) - destroy_conditionally!(merge_request) + destroy_conditionally!(merge_request) do |merge_request| + Issuable::DestroyService.new(user_project, current_user).execute(merge_request) + end end params do diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index f1eaff6b0eb..32b77aedba8 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -19,6 +19,16 @@ module API present paginate(namespaces), with: Entities::Namespace, current_user: current_user end + + desc 'Get a namespace by ID' do + success Entities::Namespace + end + params do + requires :id, type: String, desc: "Namespace's ID or path" + end + get ':id' do + present user_namespace, with: Entities::Namespace, current_user: current_user + end end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 0b9ab4eeb05..3588dc85c9e 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -18,6 +18,10 @@ module API end params do requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return notes ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return notes sorted in `asc` or `desc` order.' use :pagination end get ":id/#{noteables_str}/:noteable_id/notes" do @@ -29,11 +33,12 @@ module API # at the DB query level (which we cannot in that case), the current # page can have less elements than :per_page even if # there's more than one page. + raw_notes = noteable.notes.with_metadata.reorder(params[:order_by] => params[:sort]) notes = # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(noteable.notes) + paginate(raw_notes) .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note else @@ -50,7 +55,7 @@ module API end get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do noteable = find_project_noteable(noteables_str, params[:noteable_id]) - note = noteable.notes.find(params[:note_id]) + note = noteable.notes.with_metadata.find(params[:note_id]) can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) if can_read_note diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 259f3f34068..d7b613a717e 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -4,7 +4,6 @@ module API before do authenticate! - require_pages_enabled! end after_validation do @@ -29,10 +28,31 @@ module API end end + resource :pages do + before do + require_pages_config_enabled! + authenticated_with_full_private_access! + end + + desc "Get all pages domains" do + success Entities::PagesDomainBasic + end + params do + use :pagination + end + get "domains" do + present paginate(PagesDomain.all), with: Entities::PagesDomainBasic + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: { id: %r{[^/]+} } do + before do + require_pages_enabled! + end + desc 'Get all pages domains' do success Entities::PagesDomain end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 4cd7e714aa2..fa222bf2b1c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -79,11 +79,11 @@ module API projects = projects.with_statistics if params[:statistics] projects = projects.with_issues_enabled if params[:with_issues_enabled] projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + projects = paginate(projects) if current_user - projects = projects.includes(:route, :taggings, namespace: :route) - project_members = current_user.project_members - group_members = current_user.group_members + project_members = current_user.project_members.preload(:source, user: [notification_settings: :source]) + group_members = current_user.group_members.preload(:source, user: [notification_settings: :source]) end options = options.reverse_merge( @@ -95,7 +95,7 @@ module API ) options[:with] = Entities::BasicProjectDetails if params[:simple] - present paginate(projects), options + present options[:with].prepare_relation(projects, options), options end end @@ -367,15 +367,16 @@ module API post ":id/fork/:forked_from_id" do authenticated_as_admin! - forked_from_project = find_project!(params[:forked_from_id]) - not_found!("Source Project") unless forked_from_project + fork_from_project = find_project!(params[:forked_from_id]) - if user_project.forked_from_project.nil? - user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + not_found!("Source Project") unless fork_from_project - ::Projects::ForksCountService.new(forked_from_project).refresh_cache + result = ::Projects::ForkService.new(fork_from_project, current_user).execute(user_project) + + if result + present user_project.reload, with: Entities::Project else - render_api_error!("Project already forked", 409) + render_api_error!("Project already forked", 409) if user_project.forked? end end @@ -383,11 +384,11 @@ module API delete ":id/fork" do authorize! :remove_fork_project, user_project - if user_project.forked? - destroy_conditionally!(user_project.forked_project_link) - else - not_modified! + result = destroy_conditionally!(user_project) do + ::Projects::UnlinkForkService.new(user_project, current_user).execute end + + result ? status(204) : not_modified! end desc 'Share the project with a group' do diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb new file mode 100644 index 00000000000..6482fd94ab8 --- /dev/null +++ b/lib/api/projects_relation_builder.rb @@ -0,0 +1,34 @@ +module API + module ProjectsRelationBuilder + extend ActiveSupport::Concern + + module ClassMethods + def prepare_relation(projects_relation, options = {}) + projects_relation = preload_relation(projects_relation, options) + execute_batch_counting(projects_relation) + projects_relation + end + + def preload_relation(projects_relation, options = {}) + projects_relation + end + + def forks_counting_projects(projects_relation) + projects_relation + end + + def batch_forks_counting(projects_relation) + ::Projects::BatchForksCountService.new(forks_counting_projects(projects_relation)).refresh_cache + end + + def batch_open_issues_counting(projects_relation) + ::Projects::BatchOpenIssuesCountService.new(projects_relation).refresh_cache + end + + def execute_batch_counting(projects_relation) + batch_forks_counting(projects_relation) + batch_open_issues_counting(projects_relation) + end + end + end +end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 15fcb9e8e27..614822509f0 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -39,11 +39,11 @@ module API end params do requires :name, type: String, desc: 'The name of the protected branch' - optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER, - values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + optional :push_access_level, type: Integer, + values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to push (defaults: `40`, master access level)' - optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER, - values: ProtectedBranchAccess::ALLOWED_ACCESS_LEVELS, + optional :merge_access_level, type: Integer, + values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to merge (defaults: `40`, master access level)' end post ':id/protected_branches' do @@ -52,15 +52,13 @@ module API conflict!("Protected branch '#{params[:name]}' already exists") end - protected_branch_params = { - name: params[:name], - push_access_levels_attributes: [{ access_level: params[:push_access_level] }], - merge_access_levels_attributes: [{ access_level: params[:merge_access_level] }] - } + # Replace with `declared(params)` after updating to grape v1.0.2 + # See https://github.com/ruby-grape/grape/pull/1710 + # and https://gitlab.com/gitlab-org/gitlab-ce/issues/40843 + declared_params = params.slice("name", "push_access_level", "merge_access_level", "allowed_to_push", "allowed_to_merge") - service_args = [user_project, current_user, protected_branch_params] - - protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute + api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params) + protected_branch = api_service.create if protected_branch.persisted? present protected_branch, with: Entities::ProtectedBranch, project: user_project diff --git a/lib/api/runner.rb b/lib/api/runner.rb index a3987c560dd..80feb629d54 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -215,18 +215,20 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts_upload_path = JobArtifactUploader.artifacts_upload_path artifacts = uploaded_file(:file, artifacts_upload_path) metadata = uploaded_file(:metadata, artifacts_upload_path) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size - job.artifacts_file = artifacts - job.artifacts_metadata = metadata - job.artifacts_expire_in = params['expire_in'] || + expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in + job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, expire_in: expire_in) + job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata + job.artifacts_expire_in = expire_in + if job.save present job, with: Entities::JobRequest::Response else diff --git a/lib/api/runners.rb b/lib/api/runners.rb index d3559ef71be..996457c5dfe 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -84,6 +84,23 @@ module API destroy_conditionally!(runner) end + + desc 'List jobs running on a runner' do + success Entities::JobBasicWithProject + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES + use :pagination + end + get ':id/jobs' do + runner = get_runner(params[:id]) + authenticate_list_runners_jobs!(runner) + + jobs = RunnerJobsFinder.new(runner, params).execute + + present paginate(jobs), with: Entities::JobBasicWithProject + end end params do @@ -165,17 +182,20 @@ module API def authenticate_show_runner!(runner) return if runner.is_shared || current_user.admin? + forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_update_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_delete_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) @@ -185,6 +205,13 @@ module API forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner is locked") if runner.locked? return if current_user.admin? + + forbidden!("No access granted") unless user_can_access_runner?(runner) + end + + def authenticate_list_runners_jobs!(runner) + return if current_user.admin? + forbidden!("No access granted") unless user_can_access_runner?(runner) end diff --git a/lib/api/services.rb b/lib/api/services.rb index 6454e475036..bbcc851d07a 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -522,6 +522,12 @@ module API name: :webhook, type: String, desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + }, + { + required: false, + name: :username, + type: String, + desc: 'The username to use to post the message' } ], 'teamcity' => [ diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 851b226e9e5..cee4d309816 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -44,9 +44,11 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - mutually_exclusive :password_authentication_enabled, :signin_enabled + optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled + optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)' optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -121,6 +123,9 @@ module API end optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", @@ -135,8 +140,11 @@ module API put "application/settings" do attrs = declared_params(include_missing: false) + # support legacy names, can be removed in v5 if attrs.has_key?(:signin_enabled) - attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:signin_enabled) + elsif attrs.has_key?(:password_authentication_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end if current_settings.update_attributes(attrs) diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 00eb7c60f16..c736cc32021 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -95,6 +95,7 @@ module API put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet attrs = declared_params(include_missing: false).merge(request: request, api: true) diff --git a/lib/api/users.rb b/lib/api/users.rb index d80b364bd09..e5de31ad51b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -31,7 +31,6 @@ module API optional :location, type: String, desc: 'The location of the user' optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' - optional :skip_confirmation, type: Boolean, default: false, desc: 'Flag indicating the account is confirmed' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' optional :avatar, type: File, desc: 'Avatar image for user' all_or_none_of :extern_uid, :provider @@ -77,6 +76,8 @@ module API forbidden!("Not authorized to access /api/v4/users") unless authorized entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic + users = users.preload(:identities, :u2f_registrations) if entity == Entities::UserWithAdmin + present paginate(users), with: entity end @@ -101,6 +102,7 @@ module API requires :email, type: String, desc: 'The email of the user' optional :password, type: String, desc: 'The password of the new user' optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token' + optional :skip_confirmation, type: Boolean, desc: 'Flag indicating the account is confirmed' at_least_one_of :password, :reset_password requires :name, type: String, desc: 'The name of the user' requires :username, type: String, desc: 'The username of the user' @@ -134,6 +136,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' optional :email, type: String, desc: 'The email of the user' optional :password, type: String, desc: 'The password of the new user' + optional :skip_reconfirmation, type: Boolean, desc: 'Flag indicating the account skips the confirmation by email' optional :name, type: String, desc: 'The name of the user' optional :username, type: String, desc: 'The username of the user' use :optional_attributes diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index f493fd7c7ec..fa0bef39602 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -169,7 +169,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) - authorize!(:update_build, build) + authorize!(:erase_build, build) return forbidden!('Build is not erasable!') unless build.erasable? build.erase(erased_by: current_user) diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index ed206a6def0..0ef26aa696a 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -106,7 +106,7 @@ module API commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - notes = Note.where(commit_id: commit.id).order(:created_at) + notes = commit.notes.order(:created_at) present paginate(notes), with: ::API::Entities::CommitNote end @@ -169,10 +169,12 @@ module API if params[:path] commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index afdd7b83998..c17b6f45ed8 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -172,8 +172,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :password_authentication_enabled - expose :password_authentication_enabled, as: :signin_enabled + expose :password_authentication_enabled_for_web, as: :password_authentication_enabled + expose :password_authentication_enabled_for_web, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb index faa265f3314..c6d9957d452 100644 --- a/lib/api/v3/runners.rb +++ b/lib/api/v3/runners.rb @@ -51,6 +51,7 @@ module API helpers do def authenticate_delete_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb index 202011cfcbe..9b4ab7630fb 100644 --- a/lib/api/v3/settings.rb +++ b/lib/api/v3/settings.rb @@ -44,8 +44,8 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do @@ -131,7 +131,9 @@ module API attrs = declared_params(include_missing: false) if attrs.has_key?(:signin_enabled) - attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:signin_enabled) + elsif attrs.has_key?(:password_authentication_enabled) + attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end if current_settings.update_attributes(attrs) diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 0762fc02d70..126ec72248e 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -91,6 +91,7 @@ module API put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet attrs = declared_params(include_missing: false) @@ -113,6 +114,7 @@ module API delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :destroy_personal_snippet, snippet snippet.destroy no_content! diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index d9436e1d5e5..7a582a20056 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', ArtifactUploader.local_store_path) + super('artifacts', LegacyArtifactUploader.local_store_path) end def create_files_dir diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 3ad09a1b421..2a04c03919d 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -7,12 +7,16 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " + progress.print " * #{display_repo_path(project)} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - # Create namespace dir if missing - FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace + # Create namespace dir or hashed path if missing + if project.hashed_storage?(:repository) + FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) + else + FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace + end if empty_repo?(project) progress.puts "[SKIPPED]".color(:cyan) @@ -42,7 +46,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_repo) - progress.print " * #{wiki.full_path} ... " + progress.print " * #{display_repo_path(wiki)} ... " if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else @@ -71,7 +75,7 @@ module Backup end Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " + progress.print " * #{display_repo_path(project)} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -104,7 +108,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - progress.print " * #{wiki.full_path} ... " + progress.print " * #{display_repo_path(wiki)} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then @@ -185,16 +189,13 @@ module Backup def progress_warn(project, cmd, output) progress.puts "[WARNING] Executing #{cmd}".color(:orange) - progress.puts "Ignoring error on #{project.full_path} - #{output}".color(:orange) + progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange) end def empty_repo?(project_or_wiki) - project_or_wiki.repository.expire_exists_cache # protect backups from stale cache - project_or_wiki.repository.empty_repo? - rescue => e - progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.full_path} - #{e.message}".color(:orange) - - false + # Protect against stale caches + project_or_wiki.repository.expire_emptiness_caches + project_or_wiki.repository.empty? end def repository_storage_paths_args @@ -204,5 +205,9 @@ module Backup def progress $progress end + + def display_repo_path(project) + project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path + end end end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index e2b57adf611..d8fb7705b2a 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -11,7 +11,7 @@ module Banzai # ref - String reference. # # Returns a Project, or nil if the reference can't be found - def project_from_ref(ref) + def parent_from_ref(ref) return context[:project] unless ref Project.find_by_full_path(ref) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 9fef386de16..e7e6a90b5fd 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -82,9 +82,9 @@ module Banzai end end - def project_from_ref_cached(ref) - cached_call(:banzai_project_refs, ref) do - project_from_ref(ref) + def from_ref_cached(ref) + cached_call("banzai_#{parent_type}_refs".to_sym, ref) do + parent_from_ref(ref) end end @@ -153,15 +153,20 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| - project_path = full_project_path(namespace_ref, project_ref) - project = project_from_ref_cached(project_path) + parent_path = if parent_type == :group + full_group_path(namespace_ref) + else + full_project_path(namespace_ref, project_ref) + end - if project + parent = from_ref_cached(parent_path) + + if parent object = if link_reference - find_object_from_link_cached(project, id) + find_object_from_link_cached(parent, id) else - find_object_cached(project, id) + find_object_cached(parent, id) end end @@ -169,13 +174,13 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, project, object, link: !!link_content) + data = data_attributes_for(link_content || match, parent, object, link: !!link_content) url = if matches.names.include?("url") && matches[:url] matches[:url] else - url_for_object_cached(object, project) + url_for_object_cached(object, parent) end content = link_content || object_link_text(object, matches) @@ -213,7 +218,8 @@ module Banzai end def object_link_text(object, matches) - text = object.reference_link_text(context[:project]) + parent = context[:project] || context[:group] + text = object.reference_link_text(parent) extras = object_link_text_extras(object, matches) text += " (#{extras.join(", ")})" if extras.any? @@ -223,17 +229,24 @@ module Banzai # Returns a Hash containing all object references (e.g. issue IDs) per the # project they belong to. - def references_per_project - @references_per_project ||= begin + def references_per_parent + @references_per ||= {} + + @references_per[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } regex = Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) nodes.each do |node| node.to_html.scan(regex) do - project_path = full_project_path($~[:namespace], $~[:project]) + path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + symbol = $~[object_sym] - refs[project_path] << symbol if object_class.reference_valid?(symbol) + refs[path] << symbol if object_class.reference_valid?(symbol) end end @@ -243,35 +256,41 @@ module Banzai # Returns a Hash containing referenced projects grouped per their full # path. - def projects_per_reference - @projects_per_reference ||= begin + def parent_per_reference + @per_reference ||= {} + + @per_reference[parent_type] ||= begin refs = Set.new - references_per_project.each do |project_ref, _| - refs << project_ref + references_per_parent.each do |ref, _| + refs << ref end - find_projects_for_paths(refs.to_a).index_by(&:full_path) + find_for_paths(refs.to_a).index_by(&:full_path) end end - def projects_relation_for_paths(paths) - Project.where_full_path_in(paths).includes(:namespace) + def relation_for_paths(paths) + klass = parent_type.to_s.camelize.constantize + result = klass.where_full_path_in(paths) + return result if parent_type == :group + + result.includes(:namespace) if parent_type == :project end # Returns projects for the given paths. - def find_projects_for_paths(paths) + def find_for_paths(paths) if RequestStore.active? - cache = project_refs_cache + cache = refs_cache to_query = paths - cache.keys unless to_query.empty? - projects = projects_relation_for_paths(to_query) + records = relation_for_paths(to_query) found = [] - projects.each do |project| - ref = project.full_path - get_or_set_cache(cache, ref) { project } + records.each do |record| + ref = record.full_path + get_or_set_cache(cache, ref) { record } found << ref end @@ -283,33 +302,37 @@ module Banzai cache.slice(*paths).values.compact else - projects_relation_for_paths(paths) + relation_for_paths(paths) end end - def current_project_path - return unless project - - @current_project_path ||= project.full_path + def current_parent_path + @current_parent_path ||= parent&.full_path end def current_project_namespace_path - return unless project - - @current_project_namespace_path ||= project.namespace.full_path + @current_project_namespace_path ||= project&.namespace&.full_path end private def full_project_path(namespace, project_ref) - return current_project_path unless project_ref + return current_parent_path unless project_ref namespace_ref = namespace || current_project_namespace_path "#{namespace_ref}/#{project_ref}" end - def project_refs_cache - RequestStore[:banzai_project_refs] ||= {} + def refs_cache + RequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + end + + def parent_type + :project + end + + def parent + parent_type == :project ? project : group end end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 714e0319025..eedb95197aa 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -22,10 +22,30 @@ module Banzai end end + def referenced_merge_request_commit_shas + return [] unless noteable.is_a?(MergeRequest) + + @referenced_merge_request_commit_shas ||= begin + referenced_shas = references_per_parent.values.reduce(:|).to_a + noteable.all_commit_shas.select do |sha| + referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) } + end + end + end + def url_for_object(commit, project) h = Gitlab::Routing.url_helpers - h.project_commit_url(project, commit, - only_path: context[:only_path]) + + if referenced_merge_request_commit_shas.include?(commit.id) + h.diffs_project_merge_request_url(project, + noteable, + commit_id: commit.id, + only_path: only_path?) + else + h.project_commit_url(project, + commit, + only_path: only_path?) + end end def object_link_text_extras(object, matches) @@ -38,6 +58,16 @@ module Banzai extras end + + private + + def noteable + context[:noteable] + end + + def only_path? + context[:only_path] + end end end end diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb new file mode 100644 index 00000000000..265924abe24 --- /dev/null +++ b/lib/banzai/filter/epic_reference_filter.rb @@ -0,0 +1,12 @@ +module Banzai + module Filter + # The actual filter is implemented in the EE mixin + class EpicReferenceFilter < IssuableReferenceFilter + self.reference_type = :epic + + def self.object_class + Epic + end + end + end +end diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb new file mode 100644 index 00000000000..7addf09be73 --- /dev/null +++ b/lib/banzai/filter/issuable_reference_filter.rb @@ -0,0 +1,31 @@ +module Banzai + module Filter + class IssuableReferenceFilter < AbstractReferenceFilter + def records_per_parent + @records_per_project ||= {} + + @records_per_project[object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + parent_records(parent, record_ids).each do |record| + hash[parent][record.iid.to_i] = record + end + end + + hash + end + end + + def find_object(parent, iid) + records_per_parent[parent][iid] + end + + def parent_from_ref(ref) + parent_per_reference[ref || current_parent_path] + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index ce1ab977d3b..6877cae8c55 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -8,46 +8,24 @@ module Banzai # When external issues tracker like Jira is activated we should not # use issue reference pattern, but we should still be able # to reference issues from other GitLab projects. - class IssueReferenceFilter < AbstractReferenceFilter + class IssueReferenceFilter < IssuableReferenceFilter self.reference_type = :issue def self.object_class Issue end - def find_object(project, iid) - issues_per_project[project][iid] - end - def url_for_object(issue, project) IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path], internal: true) end - def project_from_ref(ref) - projects_per_reference[ref || current_project_path] - end - - # Returns a Hash containing the issues per Project instance. - def issues_per_project - @issues_per_project ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - projects_per_reference.each do |path, project| - issue_ids = references_per_project[path] - issues = project.issues.where(iid: issue_ids.to_a) - - issues.each do |issue| - hash[project][issue.iid.to_i] = issue - end - end - - hash - end - end - def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end + + def parent_records(parent, ids) + parent.issues.where(iid: ids.to_a) + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 5364984c9d3..d5360ad8f68 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -33,7 +33,7 @@ module Banzai end def find_label(project_ref, label_id, label_name) - project = project_from_ref(project_ref) + project = parent_from_ref(project_ref) return unless project label_params = label_params(label_id, label_name) @@ -66,7 +66,7 @@ module Banzai def object_link_text(object, matches) project_path = full_project_path(matches[:namespace], matches[:project]) - project_from_ref = project_from_ref_cached(project_path) + project_from_ref = from_ref_cached(project_path) reference = project_from_ref.to_human_reference(project) label_suffix = " <i>in #{reference}</i>" if reference.present? diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 0eab865ac04..b3cfa97d0e0 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -4,48 +4,19 @@ module Banzai # to merge requests that do not exist are ignored. # # This filter supports cross-project references. - class MergeRequestReferenceFilter < AbstractReferenceFilter + class MergeRequestReferenceFilter < IssuableReferenceFilter self.reference_type = :merge_request def self.object_class MergeRequest end - def find_object(project, iid) - merge_requests_per_project[project][iid] - end - def url_for_object(mr, project) h = Gitlab::Routing.url_helpers h.project_merge_request_url(project, mr, only_path: context[:only_path]) end - def project_from_ref(ref) - projects_per_reference[ref || current_project_path] - end - - # Returns a Hash containing the merge_requests per Project instance. - def merge_requests_per_project - @merge_requests_per_project ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - projects_per_reference.each do |path, project| - merge_request_ids = references_per_project[path] - - merge_requests = project.merge_requests - .where(iid: merge_request_ids.to_a) - .includes(target_project: :namespace) - - merge_requests.each do |merge_request| - hash[project][merge_request.iid.to_i] = merge_request - end - end - - hash - end - end - def object_link_text_extras(object, matches) extras = super @@ -61,6 +32,12 @@ module Banzai extras end + + def parent_records(parent, ids) + parent.merge_requests + .where(iid: ids.to_a) + .includes(target_project: :namespace) + end end end end diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb new file mode 100644 index 00000000000..b545b947a2c --- /dev/null +++ b/lib/banzai/filter/mermaid_filter.rb @@ -0,0 +1,20 @@ +module Banzai + module Filter + class MermaidFilter < HTML::Pipeline::Filter + def call + doc.css('pre[lang="mermaid"]').add_class('mermaid') + doc.css('pre[lang="mermaid"]').add_class('js-render-mermaid') + + # The `<code></code>` blocks are added in the lib/banzai/filter/syntax_highlight_filter.rb + # We want to keep context and consistency, so we the blocks are added for all filters. + # Details: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15107/diffs?diff_id=7962900#note_45495859 + doc.css('pre[lang="mermaid"]').each do |pre| + document = pre.at('code') + document.replace(document.content) + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index bb5da310e09..2a6b0964ac5 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -38,7 +38,7 @@ module Banzai def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) project_path = full_project_path(namespace_ref, project_ref) - project = project_from_ref(project_path) + project = parent_from_ref(project_path) return unless project diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 7da565043d1..a79a0154846 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,23 +14,26 @@ module Banzai end def highlight_node(node) - language = node.attr('lang') code = node.text - css_classes = "code highlight" - lexer = lexer_for(language) - lang = lexer.tag - - begin - code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: lang) - - css_classes << " js-syntax-highlight #{lang}" - rescue - lang = nil - # Gracefully handle syntax highlighter bugs/errors to ensure - # users can still access an issue/comment/etc. + css_classes = 'code highlight js-syntax-highlight' + language = node.attr('lang') + + if use_rouge?(language) + lexer = lexer_for(language) + language = lexer.tag + + begin + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: language) + css_classes << " #{language}" + rescue + # Gracefully handle syntax highlighter bugs/errors to ensure + # users can still access an issue/comment/etc. + + language = nil + end end - highlighted = %(<pre class="#{css_classes}" lang="#{lang}" v-pre="true"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" lang="#{language}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) @@ -51,6 +54,10 @@ module Banzai # Replace the parent `pre` element with the entire highlighted block node.parent.replace(highlighted) end + + def use_rouge?(language) + %w(math mermaid plantuml).exclude?(language) + end end end end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 09844931be5..d64f9ac4eb6 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -8,7 +8,7 @@ module Banzai # class UploadLinkFilter < HTML::Pipeline::Filter def call - return doc unless project + return doc unless project || group doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el| process_link_attr el.attribute('href') @@ -28,13 +28,27 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, project.full_path, uri) + base_path = Gitlab.config.gitlab.url + + if group + urls = Gitlab::Routing.url_helpers + # we need to get last 2 parts of the uri which are secret and filename + uri_parts = uri.split(File::SEPARATOR) + file_path = urls.show_group_uploads_path(group, uri_parts[-2], uri_parts[-1]) + File.join(base_path, file_path) + else + File.join(base_path, project.full_path, uri) + end end def project context[:project] end + def group + context[:group] + end + # Ensure that a :project key exists in context # # Note that while the key might exist, its value could be nil! diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index cbabf9156de..49603d0b363 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -28,8 +28,8 @@ module Banzai issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) - issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( - merge_request_parser.merge_requests_for_nodes(nodes) + issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( + merge_request_parser.records_for_nodes(nodes) ) # The project for the issue/MR might be pending for deletion! diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9bb8ed913d8..2691be81623 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -17,11 +17,11 @@ module Banzai # project - A Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. - # context - A Hash containing extra attributes to use during redaction + # redaction_context - A Hash containing extra attributes to use during redaction def initialize(project, user = nil, redaction_context = {}) @project = project @user = user - @redaction_context = redaction_context + @redaction_context = base_context.merge(redaction_context) end # Renders and redacts an Array of objects. @@ -73,19 +73,20 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - base_context.merge(object.banzai_render_context(attribute)) + @redaction_context.merge(object.banzai_render_context(attribute)) end def base_context - @base_context ||= @redaction_context.merge( + { current_user: user, project: project, skip_redaction: true - ) + } end def save_options - return {} unless base_context[:xhtml] + return {} unless @redaction_context[:xhtml] + { save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML } end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 3208abfc538..55874ad50a3 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -14,6 +14,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::MathFilter, + Filter::MermaidFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb index fb2faae02bc..a19a05e8c0d 100644 --- a/lib/banzai/querying.rb +++ b/lib/banzai/querying.rb @@ -52,8 +52,10 @@ module Banzai children.each do |child| next if child.text.blank? + node = nodes.shift break unless node == child + filtered_nodes << node end end diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb new file mode 100644 index 00000000000..08b8a4c9a0f --- /dev/null +++ b/lib/banzai/reference_parser/epic_parser.rb @@ -0,0 +1,12 @@ +module Banzai + module ReferenceParser + # The actual parser is implemented in the EE mixin + class EpicParser < IssuableParser + self.reference_type = :epic + + def records_for_nodes(_nodes) + {} + end + end + end +end diff --git a/lib/banzai/reference_parser/issuable_parser.rb b/lib/banzai/reference_parser/issuable_parser.rb new file mode 100644 index 00000000000..3953867eb83 --- /dev/null +++ b/lib/banzai/reference_parser/issuable_parser.rb @@ -0,0 +1,25 @@ +module Banzai + module ReferenceParser + class IssuableParser < BaseParser + def nodes_visible_to_user(user, nodes) + records = records_for_nodes(nodes) + + nodes.select do |node| + issuable = records[node] + + issuable && can_read_reference?(user, issuable) + end + end + + def referenced_by(nodes) + records = records_for_nodes(nodes) + + nodes.map { |node| records[node] }.compact.uniq + end + + def can_read_reference?(user, issuable) + can?(user, "read_#{issuable.class.to_s.underscore}".to_sym, issuable) + end + end + end +end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index e0a8ca653cb..38d4e3f3e44 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -1,10 +1,10 @@ module Banzai module ReferenceParser - class IssueParser < BaseParser + class IssueParser < IssuableParser self.reference_type = :issue def nodes_visible_to_user(user, nodes) - issues = issues_for_nodes(nodes) + issues = records_for_nodes(nodes) readable_issues = Ability .issues_readable_by_user(issues.values, user).to_set @@ -14,13 +14,7 @@ module Banzai end end - def referenced_by(nodes) - issues = issues_for_nodes(nodes) - - nodes.map { |node| issues[node] }.compact.uniq - end - - def issues_for_nodes(nodes) + def records_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, Issue.all.includes( diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 75cbc7fdac4..a370ff5b5b3 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -1,25 +1,9 @@ module Banzai module ReferenceParser - class MergeRequestParser < BaseParser + class MergeRequestParser < IssuableParser self.reference_type = :merge_request - def nodes_visible_to_user(user, nodes) - merge_requests = merge_requests_for_nodes(nodes) - - nodes.select do |node| - merge_request = merge_requests[node] - - merge_request && can?(user, :read_merge_request, merge_request.project) - end - end - - def referenced_by(nodes) - merge_requests = merge_requests_for_nodes(nodes) - - nodes.map { |node| merge_requests[node] }.compact.uniq - end - - def merge_requests_for_nodes(nodes) + def records_for_nodes(nodes) @merge_requests_for_nodes ||= grouped_objects_for_nodes( nodes, MergeRequest.includes( @@ -40,10 +24,6 @@ module Banzai self.class.data_attribute ) end - - def can_read_reference?(user, ref_project, node) - can?(user, :read_merge_request, ref_project) - end end end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 4d336068861..8932d4f2905 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -31,6 +31,7 @@ module Banzai nodes.each do |node| if node.has_attribute?(group_attr) next unless can_read_group_reference?(node, user, groups) + visible << node elsif can_read_project_reference?(node) visible << node diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 5cb9adf52b0..0050295eeda 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -149,6 +149,7 @@ module Banzai def self.full_cache_key(cache_key, pipeline_name) return unless cache_key + ["banzai", *cache_key, pipeline_name || :full] end @@ -157,6 +158,7 @@ module Banzai # method. def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key + Rails.cache.__send__(:expanded_key, full_cache_key(cache_key, pipeline_name)) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index ae65653645b..b1949d693ad 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -30,6 +30,7 @@ module DeclarativePolicy policy_class = class_for_class(subject.class) raise "no policy for #{subject.class.name}" if policy_class.nil? + policy_class end @@ -84,6 +85,7 @@ module DeclarativePolicy while subject.respond_to?(:declarative_policy_delegate) raise ArgumentError, "circular delegations" if seen.include?(subject.object_id) + seen << subject.object_id subject = subject.declarative_policy_delegate end diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index b028169f500..47542194497 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -276,6 +276,7 @@ module DeclarativePolicy # boolean `false` def cache(key, &b) return @cache[key] if cached?(key) + @cache[key] = yield end @@ -291,6 +292,7 @@ module DeclarativePolicy @_conditions[name] ||= begin raise "invalid condition #{name}" unless self.class.conditions.key?(name) + ManifestCondition.new(self.class.conditions[name], self) end end diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb index 0804edba016..780d8f707bd 100644 --- a/lib/declarative_policy/cache.rb +++ b/lib/declarative_policy/cache.rb @@ -3,6 +3,7 @@ module DeclarativePolicy class << self def user_key(user) return '<anonymous>' if user.nil? + id_for(user) end @@ -15,6 +16,7 @@ module DeclarativePolicy def subject_key(subject) return '<nil>' if subject.nil? return subject.inspect if subject.is_a?(Symbol) + "#{subject.class.name}:#{id_for(subject)}" end diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index 7cfa82a9a9f..e309244a3b3 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -83,6 +83,7 @@ module DeclarativePolicy def cached_pass?(context) condition = context.condition(@name) return nil unless condition.cached? + condition.pass? end @@ -109,6 +110,7 @@ module DeclarativePolicy def delegated_context(context) policy = context.delegated_policies[@delegate_name] raise MissingDelegate if policy.nil? + policy end @@ -121,6 +123,7 @@ module DeclarativePolicy def cached_pass?(context) condition = delegated_context(context).condition(@name) return nil unless condition.cached? + condition.pass? rescue MissingDelegate false @@ -157,6 +160,7 @@ module DeclarativePolicy def cached_pass?(context) runner = context.runner(@ability) return nil unless runner.cached? + runner.pass? end @@ -258,6 +262,7 @@ module DeclarativePolicy def score(context) return 0 unless cached_pass?(context).nil? + @rules.map { |r| r.score(context) }.inject(0, :+) end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 45ff2ef9ced..77c91817382 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -43,6 +43,7 @@ module DeclarativePolicy # used by Rule::Ability. See #steps_by_score def score return 0 if cached? + steps.map(&:score).inject(0, :+) end diff --git a/lib/feature.rb b/lib/feature.rb index 4bd29aed687..ac3bc65c0d5 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -5,6 +5,10 @@ class Feature class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature # Using `self.table_name` won't work. ActiveRecord bug? superclass.table_name = 'features' + + def self.feature_names + pluck(:key) + end end class FlipperGate < Flipper::Adapters::ActiveRecord::Gate @@ -22,11 +26,19 @@ class Feature flipper.feature(key) end + def persisted_names + if RequestStore.active? + RequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names + else + FlipperFeature.feature_names + end + end + def persisted?(feature) # Flipper creates on-memory features when asked for a not-yet-created one. # If we want to check if a feature has been actually set, we look for it # on the persisted features list. - all.map(&:name).include?(feature.name) + persisted_names.include?(feature.name) end def enabled?(key, thing = nil) diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index de391de9059..69d981e8be9 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -8,6 +8,7 @@ class FileSizeValidator < ActiveModel::EachValidator def initialize(options) if range = (options.delete(:in) || options.delete(:within)) raise ArgumentError, ":in and :within must be a Range" unless range.is_a?(Range) + options[:minimum], options[:maximum] = range.begin, range.end options[:maximum] -= 1 if range.exclude_end? end diff --git a/lib/github/client.rb b/lib/github/client.rb deleted file mode 100644 index 29bd9c1f39e..00000000000 --- a/lib/github/client.rb +++ /dev/null @@ -1,54 +0,0 @@ -module Github - class Client - TIMEOUT = 60 - DEFAULT_PER_PAGE = 100 - - attr_reader :connection, :rate_limit - - def initialize(options) - @connection = Faraday.new(url: options.fetch(:url, root_endpoint)) do |faraday| - faraday.options.open_timeout = options.fetch(:timeout, TIMEOUT) - faraday.options.timeout = options.fetch(:timeout, TIMEOUT) - faraday.authorization 'token', options.fetch(:token) - faraday.adapter :net_http - faraday.ssl.verify = verify_ssl - end - - @rate_limit = RateLimit.new(connection) - end - - def get(url, query = {}) - exceed, reset_in = rate_limit.get - sleep reset_in if exceed - - Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query))) - end - - private - - def root_endpoint - custom_endpoint || github_endpoint - end - - def custom_endpoint - github_omniauth_provider.dig('args', 'client_options', 'site') - end - - def verify_ssl - # If there is no config, we're connecting to github.com - # and we should verify ssl. - github_omniauth_provider.fetch('verify_ssl', true) - end - - def github_endpoint - OmniAuth::Strategies::GitHub.default_options[:client_options][:site] - end - - def github_omniauth_provider - @github_omniauth_provider ||= - Gitlab.config.omniauth.providers - .find { |provider| provider.name == 'github' } - .to_h - end - end -end diff --git a/lib/github/collection.rb b/lib/github/collection.rb deleted file mode 100644 index 014b2038c4b..00000000000 --- a/lib/github/collection.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Github - class Collection - attr_reader :options - - def initialize(options) - @options = options - end - - def fetch(url, query = {}) - return [] if url.blank? - - Enumerator.new do |yielder| - loop do - response = client.get(url, query) - response.body.each { |item| yielder << item } - - raise StopIteration unless response.rels.key?(:next) - url = response.rels[:next] - end - end.lazy - end - - private - - def client - @client ||= Github::Client.new(options) - end - end -end diff --git a/lib/github/error.rb b/lib/github/error.rb deleted file mode 100644 index 66d7afaa787..00000000000 --- a/lib/github/error.rb +++ /dev/null @@ -1,3 +0,0 @@ -module Github - RepositoryFetchError = Class.new(StandardError) -end diff --git a/lib/github/import.rb b/lib/github/import.rb deleted file mode 100644 index fef63dd7168..00000000000 --- a/lib/github/import.rb +++ /dev/null @@ -1,377 +0,0 @@ -require_relative 'error' -require_relative 'import/issue' -require_relative 'import/legacy_diff_note' -require_relative 'import/merge_request' -require_relative 'import/note' - -module Github - class Import - include Gitlab::ShellAdapter - - attr_reader :project, :repository, :repo, :repo_url, :wiki_url, - :options, :errors, :cached, :verbose, :last_fetched_at - - def initialize(project, options = {}) - @project = project - @repository = project.repository - @repo = project.import_source - @repo_url = project.import_url - @wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') - @options = options.reverse_merge(token: project.import_data&.credentials&.fetch(:user)) - @verbose = options.fetch(:verbose, false) - @cached = Hash.new { |hash, key| hash[key] = Hash.new } - @errors = [] - @last_fetched_at = nil - end - - # rubocop: disable Rails/Output - def execute - puts 'Fetching repository...'.color(:aqua) if verbose - setup_and_fetch_repository - puts 'Fetching labels...'.color(:aqua) if verbose - fetch_labels - puts 'Fetching milestones...'.color(:aqua) if verbose - fetch_milestones - puts 'Fetching pull requests...'.color(:aqua) if verbose - fetch_pull_requests - puts 'Fetching issues...'.color(:aqua) if verbose - fetch_issues - puts 'Fetching releases...'.color(:aqua) if verbose - fetch_releases - puts 'Cloning wiki repository...'.color(:aqua) if verbose - fetch_wiki_repository - puts 'Expiring repository cache...'.color(:aqua) if verbose - expire_repository_cache - - errors.empty? - rescue Github::RepositoryFetchError - expire_repository_cache - false - ensure - keep_track_of_errors - end - - private - - def setup_and_fetch_repository - begin - project.ensure_repository - project.repository.add_remote('github', repo_url) - project.repository.set_import_remote_as_mirror('github') - project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') - fetch_remote(forced: true) - rescue Gitlab::Git::Repository::NoRepository, - Gitlab::Git::RepositoryMirroring::RemoteError, - Gitlab::Shell::Error => e - error(:project, repo_url, e.message) - raise Github::RepositoryFetchError - end - end - - def fetch_remote(forced: false) - @last_fetched_at = Time.now - project.repository.fetch_remote('github', forced: forced) - end - - def fetch_wiki_repository - return if project.wiki.repository_exists? - - wiki_path = project.wiki.disk_path - gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) - rescue Gitlab::Shell::Error => e - # GitHub error message when the wiki repo has not been created, - # this means that repo has wiki enabled, but have no pages. So, - # we can skip the import. - if e.message !~ /repository not exported/ - error(:wiki, wiki_url, e.message) - end - end - - def fetch_labels - url = "/repos/#{repo}/labels" - - while url - response = Github::Client.new(options).get(url) - - response.body.each do |raw| - begin - representation = Github::Representation::Label.new(raw) - - label = project.labels.find_or_create_by!(title: representation.title) do |label| - label.color = representation.color - end - - cached[:label_ids][representation.title] = label.id - rescue => e - error(:label, representation.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_milestones - url = "/repos/#{repo}/milestones" - - while url - response = Github::Client.new(options).get(url, state: :all) - - response.body.each do |raw| - begin - milestone = Github::Representation::Milestone.new(raw) - next if project.milestones.where(iid: milestone.iid).exists? - - project.milestones.create!( - iid: milestone.iid, - title: milestone.title, - description: milestone.description, - due_date: milestone.due_date, - state: milestone.state, - created_at: milestone.created_at, - updated_at: milestone.updated_at - ) - rescue => e - error(:milestone, milestone.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_pull_requests - url = "/repos/#{repo}/pulls" - - while url - response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - - response.body.each do |raw| - pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) - merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) - next unless merge_request.new_record? && pull_request.valid? - - begin - # If the PR has been created/updated after we last fetched the - # remote, we fetch again to get the up-to-date refs. - fetch_remote if pull_request.updated_at > last_fetched_at - - author_id = user_id(pull_request.author, project.creator_id) - description = format_description(pull_request.description, pull_request.author) - - merge_request.attributes = { - iid: pull_request.iid, - title: pull_request.title, - description: description, - source_project: pull_request.source_project, - source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, - target_project: pull_request.target_project, - target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, - state: pull_request.state, - milestone_id: milestone_id(pull_request.milestone), - author_id: author_id, - assignee_id: user_id(pull_request.assignee), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - } - - merge_request.save!(validate: false) - merge_request.merge_request_diffs.create - - review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" - fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) - rescue => e - error(:pull_request, pull_request.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_issues - url = "/repos/#{repo}/issues" - - while url - response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - - response.body.each { |raw| populate_issue(raw) } - - url = response.rels[:next] - end - end - - def populate_issue(raw) - representation = Github::Representation::Issue.new(raw, options) - - begin - # Every pull request is an issue, but not every issue - # is a pull request. For this reason, "shared" actions - # for both features, like manipulating assignees, labels - # and milestones, are provided within the Issues API. - if representation.pull_request? - return unless representation.labels? || representation.comments? - - merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - - if representation.labels? - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) - end - - fetch_comments_conditionally(merge_request, representation) - else - return if Issue.exists?(iid: representation.iid, project_id: project.id) - - author_id = user_id(representation.author, project.creator_id) - issue = Issue.new - issue.iid = representation.iid - issue.project_id = project.id - issue.title = representation.title - issue.description = format_description(representation.description, representation.author) - issue.state = representation.state - issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = author_id - issue.created_at = representation.created_at - issue.updated_at = representation.updated_at - issue.save!(validate: false) - - issue.update( - label_ids: label_ids(representation.labels), - assignee_ids: assignee_ids(representation.assignees)) - - fetch_comments_conditionally(issue, representation) - end - rescue => e - error(:issue, representation.url, e.message) - end - end - - def fetch_comments_conditionally(issuable, representation) - if representation.comments? - comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments" - fetch_comments(issuable, :comment, comments_url) - end - end - - def fetch_comments(noteable, type, url, klass = Note) - while url - comments = Github::Client.new(options).get(url) - - ActiveRecord::Base.no_touching do - comments.body.each do |raw| - begin - representation = Github::Representation::Comment.new(raw, options) - author_id = user_id(representation.author, project.creator_id) - - note = klass.new - note.project_id = project.id - note.noteable = noteable - note.note = format_description(representation.note, representation.author) - note.commit_id = representation.commit_id - note.line_code = representation.line_code - note.author_id = author_id - note.created_at = representation.created_at - note.updated_at = representation.updated_at - note.save!(validate: false) - rescue => e - error(type, representation.url, e.message) - end - end - end - - url = comments.rels[:next] - end - end - - def fetch_releases - url = "/repos/#{repo}/releases" - - while url - response = Github::Client.new(options).get(url) - - response.body.each do |raw| - representation = Github::Representation::Release.new(raw) - next unless representation.valid? - - release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) - next unless release.new_record? - - begin - release.description = representation.description - release.created_at = representation.created_at - release.updated_at = representation.updated_at - release.save!(validate: false) - rescue => e - error(:release, representation.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def label_ids(labels) - labels.map { |label| cached[:label_ids][label.title] }.compact - end - - def assignee_ids(assignees) - assignees.map { |assignee| user_id(assignee) }.compact - end - - def milestone_id(milestone) - return unless milestone.present? - - project.milestones.select(:id).find_by(iid: milestone.iid)&.id - end - - def user_id(user, fallback_id = nil) - return unless user.present? - return cached[:user_ids][user.id] if cached[:user_ids][user.id].present? - - gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) - - cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? - cached[:user_ids][user.id] = gitlab_user_id || fallback_id - end - - def user_id_by_email(email) - return nil unless email - - ::User.find_by_any_email(email)&.id - end - - def user_id_by_external_uid(id) - return nil unless id - - ::User.select(:id) - .joins(:identities) - .merge(::Identity.where(provider: :github, extern_uid: id)) - .first&.id - end - - def format_description(body, author) - return body if cached[:gitlab_user_ids][author.id] - - "*Created by: #{author.username}*\n\n#{body}" - end - - def expire_repository_cache - repository.expire_content_cache if project.repository_exists? - end - - def keep_track_of_errors - return unless errors.any? - - project.update_column(:import_error, { - message: 'The remote data could not be fully imported.', - errors: errors - }.to_json) - end - - def error(type, url, message) - errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } - end - end -end diff --git a/lib/github/import/issue.rb b/lib/github/import/issue.rb deleted file mode 100644 index 171f0872666..00000000000 --- a/lib/github/import/issue.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class Issue < ::Issue - self.table_name = 'issues' - - self.reset_callbacks :save - self.reset_callbacks :create - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/legacy_diff_note.rb b/lib/github/import/legacy_diff_note.rb deleted file mode 100644 index 18adff560b6..00000000000 --- a/lib/github/import/legacy_diff_note.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Github - class Import - class LegacyDiffNote < ::LegacyDiffNote - self.table_name = 'notes' - self.store_full_sti_class = false - - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/merge_request.rb b/lib/github/import/merge_request.rb deleted file mode 100644 index c258e5d5e0e..00000000000 --- a/lib/github/import/merge_request.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class MergeRequest < ::MergeRequest - self.table_name = 'merge_requests' - - self.reset_callbacks :create - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/note.rb b/lib/github/import/note.rb deleted file mode 100644 index 8cf4f30e6b7..00000000000 --- a/lib/github/import/note.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class Note < ::Note - self.table_name = 'notes' - self.store_full_sti_class = false - - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb deleted file mode 100644 index 884693d093c..00000000000 --- a/lib/github/rate_limit.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Github - class RateLimit - SAFE_REMAINING_REQUESTS = 100 - SAFE_RESET_TIME = 500 - RATE_LIMIT_URL = '/rate_limit'.freeze - - attr_reader :connection - - def initialize(connection) - @connection = connection - end - - def get - response = connection.get(RATE_LIMIT_URL) - - # GitHub Rate Limit API returns 404 when the rate limit is disabled - return false unless response.status != 404 - - body = Oj.load(response.body, class_cache: false, mode: :compat) - remaining = body.dig('rate', 'remaining').to_i - reset_in = body.dig('rate', 'reset').to_i - exceed = remaining <= SAFE_REMAINING_REQUESTS - - [exceed, reset_in] - end - end -end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb deleted file mode 100644 index c1c9448f305..00000000000 --- a/lib/github/repositories.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Github - class Repositories - attr_reader :options - - def initialize(options) - @options = options - end - - def fetch - Collection.new(options).fetch(repos_url) - end - - private - - def repos_url - '/user/repos' - end - end -end diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb deleted file mode 100644 index f26bdbdd546..00000000000 --- a/lib/github/representation/base.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Github - module Representation - class Base - def initialize(raw, options = {}) - @raw = raw - @options = options - end - - def id - raw['id'] - end - - def url - raw['url'] - end - - def created_at - raw['created_at'] - end - - def updated_at - raw['updated_at'] - end - - private - - attr_reader :raw, :options - end - end -end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb deleted file mode 100644 index 0087a3d3c4f..00000000000 --- a/lib/github/representation/branch.rb +++ /dev/null @@ -1,55 +0,0 @@ -module Github - module Representation - class Branch < Representation::Base - attr_reader :repository - - def user - raw.dig('user', 'login') || 'unknown' - end - - def repo? - raw['repo'].present? - end - - def repo - return unless repo? - - @repo ||= Github::Representation::Repo.new(raw['repo']) - end - - def ref - raw['ref'] - end - - def sha - raw['sha'] - end - - def short_sha - Commit.truncate_sha(sha) - end - - def valid? - sha.present? && ref.present? - end - - def restore!(name) - repository.create_branch(name, sha) - rescue Gitlab::Git::Repository::InvalidRef => e - Rails.logger.error("#{self.class.name}: Could not restore branch #{name}: #{e}") - end - - def remove!(name) - repository.delete_branch(name) - rescue Gitlab::Git::Repository::DeleteBranchError => e - Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}") - end - - private - - def repository - @repository ||= options.fetch(:repository) - end - end - end -end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb deleted file mode 100644 index 83bf0b5310d..00000000000 --- a/lib/github/representation/comment.rb +++ /dev/null @@ -1,42 +0,0 @@ -module Github - module Representation - class Comment < Representation::Base - def note - raw['body'] || '' - end - - def author - @author ||= Github::Representation::User.new(raw['user'], options) - end - - def commit_id - raw['commit_id'] - end - - def line_code - return unless on_diff? - - parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) - generate_line_code(parsed_lines.to_a.last) - end - - private - - def generate_line_code(line) - Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) - end - - def on_diff? - diff_hunk.present? - end - - def diff_hunk - raw['diff_hunk'] - end - - def file_path - raw['path'] - end - end - end -end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb deleted file mode 100644 index 768ba3b993c..00000000000 --- a/lib/github/representation/issuable.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Github - module Representation - class Issuable < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['body'] || '' - end - - def milestone - return unless raw['milestone'].present? - - @milestone ||= Github::Representation::Milestone.new(raw['milestone']) - end - - def author - @author ||= Github::Representation::User.new(raw['user'], options) - end - - def labels? - raw['labels'].any? - end - - def labels - @labels ||= Array(raw['labels']).map do |label| - Github::Representation::Label.new(label, options) - end - end - end - end -end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb deleted file mode 100644 index 4f1a02cb90f..00000000000 --- a/lib/github/representation/issue.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Github - module Representation - class Issue < Representation::Issuable - def state - raw['state'] == 'closed' ? 'closed' : 'opened' - end - - def comments? - raw['comments'] > 0 - end - - def pull_request? - raw['pull_request'].present? - end - - def assigned? - raw['assignees'].present? - end - - def assignees - @assignees ||= Array(raw['assignees']).map do |user| - Github::Representation::User.new(user, options) - end - end - end - end -end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb deleted file mode 100644 index 60aa51f9569..00000000000 --- a/lib/github/representation/label.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - module Representation - class Label < Representation::Base - def color - "##{raw['color']}" - end - - def title - raw['name'] - end - end - end -end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb deleted file mode 100644 index 917e6394ad4..00000000000 --- a/lib/github/representation/milestone.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Github - module Representation - class Milestone < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['description'] - end - - def due_date - raw['due_on'] - end - - def state - raw['state'] == 'closed' ? 'closed' : 'active' - end - end - end -end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb deleted file mode 100644 index 0171179bb0f..00000000000 --- a/lib/github/representation/pull_request.rb +++ /dev/null @@ -1,71 +0,0 @@ -module Github - module Representation - class PullRequest < Representation::Issuable - delegate :sha, to: :source_branch, prefix: true - delegate :sha, to: :target_branch, prefix: true - - def source_project - project - end - - def source_branch_name - # Mimic the "user:branch" displayed in the MR widget, - # i.e. "Request to merge rymai:add-external-mounts into master" - cross_project? ? "#{source_branch.user}:#{source_branch.ref}" : source_branch.ref - end - - def target_project - project - end - - def target_branch_name - target_branch.ref - end - - def state - return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? - return 'closed' if raw['state'] == 'closed' - - 'opened' - end - - def opened? - state == 'opened' - end - - def valid? - source_branch.valid? && target_branch.valid? - end - - def assigned? - raw['assignee'].present? - end - - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee'], options) - end - - private - - def project - @project ||= options.fetch(:project) - end - - def source_branch - @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) - end - - def target_branch - @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) - end - - def cross_project? - return true unless source_branch.repo? - - source_branch.repo.id != target_branch.repo.id - end - end - end -end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb deleted file mode 100644 index e7e4b428c1a..00000000000 --- a/lib/github/representation/release.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Github - module Representation - class Release < Representation::Base - def description - raw['body'] - end - - def tag - raw['tag_name'] - end - - def valid? - !raw['draft'] - end - end - end -end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb deleted file mode 100644 index 6938aa7db05..00000000000 --- a/lib/github/representation/repo.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Github - module Representation - class Repo < Representation::Base - end - end -end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb deleted file mode 100644 index 18591380e25..00000000000 --- a/lib/github/representation/user.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Github - module Representation - class User < Representation::Base - def email - return @email if defined?(@email) - - @email = Github::User.new(username, options).get.fetch('email', nil) - end - - def username - raw['login'] - end - end - end -end diff --git a/lib/github/response.rb b/lib/github/response.rb deleted file mode 100644 index 761c524b553..00000000000 --- a/lib/github/response.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Github - class Response - attr_reader :raw, :headers, :status - - def initialize(response) - @raw = response - @headers = response.headers - @status = response.status - end - - def body - Oj.load(raw.body, class_cache: false, mode: :compat) - end - - def rels - links = headers['Link'].to_s.split(', ').map do |link| - href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures - - [name.to_sym, href] - end - - Hash[*links.flatten] - end - end -end diff --git a/lib/github/user.rb b/lib/github/user.rb deleted file mode 100644 index f88a29e590b..00000000000 --- a/lib/github/user.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Github - class User - attr_reader :username, :options - - def initialize(username, options) - @username = username - @options = options - end - - def get - client.get(user_url).body - end - - private - - def client - @client ||= Github::Client.new(options) - end - - def user_url - "/users/#{username}" - end - end -end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index b4012ebbb99..7127948cf00 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -58,9 +58,9 @@ module Gitlab def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, - "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, - "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL + "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Masters can push to the branch." => PROTECTION_DEV_CAN_MERGE, + "Partially protected: Both developers and masters can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH, + "Fully protected: Developers cannot push new commits, but masters can. No-one can force push or delete the branch." => PROTECTION_FULL } end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 0ad9285c0ea..65d7fd3ec70 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -25,7 +25,7 @@ module Gitlab result = service_request_check(login, password, project) || build_access_token_check(login, password) || - lfs_token_check(login, password) || + lfs_token_check(login, password, project) || oauth_access_token_check(login, password) || personal_access_token_check(password) || user_with_password_for_git(login, password) || @@ -34,7 +34,7 @@ module Gitlab rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + return result if result.success? || authenticate_using_internal_or_ldap_password? # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts @@ -45,6 +45,10 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless authenticate_using_internal_or_ldap_password? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -52,10 +56,8 @@ module Gitlab # LDAP users are only authenticated via LDAP if user.nil? || user.ldap_user? # Second chance - try LDAP authentication - return unless Gitlab::LDAP::Config.enabled? - Gitlab::LDAP::Authentication.login(login, password) - else + elsif current_application_settings.password_authentication_enabled_for_git? user if user.active? && user.valid_password?(password) end end @@ -84,6 +86,10 @@ module Gitlab private + def authenticate_using_internal_or_ldap_password? + current_application_settings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + end + def service_request_check(login, password, project) matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) @@ -128,7 +134,7 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) if token && valid_scoped_token?(token, available_scopes) - Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scope(token.scopes)) + Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end @@ -140,13 +146,18 @@ module Gitlab AccessTokenValidationService.new(token).include_any_scope?(scopes) end - def abilities_for_scope(scopes) - scopes.map do |scope| - self.public_send(:"#{scope}_scope_authentication_abilities") # rubocop:disable GitlabSecurity/PublicSend - end.flatten.uniq + def abilities_for_scopes(scopes) + abilities_by_scope = { + api: full_authentication_abilities, + read_registry: [:read_container_image] + } + + scopes.flat_map do |scope| + abilities_by_scope.fetch(scope.to_sym, []) + end.uniq end - def lfs_token_check(login, password) + def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) actor = @@ -163,6 +174,8 @@ module Gitlab authentication_abilities = if token_handler.user? full_authentication_abilities + elsif token_handler.deploy_key_pushable?(project) + read_write_authentication_abilities else read_authentication_abilities end @@ -208,22 +221,17 @@ module Gitlab ] end - def full_authentication_abilities + def read_write_authentication_abilities read_authentication_abilities + [ :push_code, - :create_container_image, - :admin_container_image + :create_container_image ] end - alias_method :api_scope_authentication_abilities, :full_authentication_abilities - - def read_registry_scope_authentication_abilities - [:read_container_image] - end - # The currently used auth method doesn't allow any actions for this scope - def read_user_scope_authentication_abilities - [] + def full_authentication_abilities + read_write_authentication_abilities + [ + :admin_container_image + ] end def available_scopes(current_user = nil) diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb new file mode 100644 index 00000000000..46ec040ce92 --- /dev/null +++ b/lib/gitlab/auth/request_authenticator.rb @@ -0,0 +1,25 @@ +# Use for authentication only, in particular for Rack::Attack. +# Does not perform authorization of scopes, etc. +module Gitlab + module Auth + class RequestAuthenticator + include UserAuthFinders + + attr_reader :request + + def initialize(request) + @request = request + end + + def user + find_sessionless_user || find_user_from_warden + end + + def find_sessionless_user + find_user_from_access_token || find_user_from_rss_token + rescue Gitlab::Auth::AuthenticationError + nil + end + end + end +end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb new file mode 100644 index 00000000000..b4114a3ac96 --- /dev/null +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -0,0 +1,109 @@ +module Gitlab + module Auth + # + # Exceptions + # + + AuthenticationError = Class.new(StandardError) + MissingTokenError = Class.new(AuthenticationError) + TokenNotFoundError = Class.new(AuthenticationError) + ExpiredError = Class.new(AuthenticationError) + RevokedError = Class.new(AuthenticationError) + UnauthorizedError = Class.new(AuthenticationError) + + class InsufficientScopeError < AuthenticationError + attr_reader :scopes + def initialize(scopes) + @scopes = scopes.map { |s| s.try(:name) || s } + end + end + + module UserAuthFinders + include Gitlab::Utils::StrongMemoize + + PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'.freeze + PRIVATE_TOKEN_PARAM = :private_token + + # Check the Rails session for valid authentication details + def find_user_from_warden + current_request.env['warden']&.authenticate if verified_request? + end + + def find_user_from_rss_token + return unless current_request.path.ends_with?('.atom') || current_request.format.atom? + + token = current_request.params[:rss_token].presence + return unless token + + User.find_by_rss_token(token) || raise(UnauthorizedError) + end + + def find_user_from_access_token + return unless access_token + + validate_access_token! + + access_token.user || raise(UnauthorizedError) + end + + def validate_access_token!(scopes: []) + return unless access_token + + case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + when AccessTokenValidationService::EXPIRED + raise ExpiredError + when AccessTokenValidationService::REVOKED + raise RevokedError + end + end + + private + + def access_token + strong_memoize(:access_token) do + find_oauth_access_token || find_personal_access_token + end + end + + def find_personal_access_token + token = + current_request.params[PRIVATE_TOKEN_PARAM].presence || + current_request.env[PRIVATE_TOKEN_HEADER].presence + + return unless token + + # Expiration, revocation and scopes are verified in `validate_access_token!` + PersonalAccessToken.find_by(token: token) || raise(UnauthorizedError) + end + + def find_oauth_access_token + token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + return unless token + + # Expiration, revocation and scopes are verified in `validate_access_token!` + oauth_token = OauthAccessToken.by_token(token) + raise UnauthorizedError unless oauth_token + + oauth_token.revoke_previous_refresh_token! + oauth_token + end + + # Check if the request is GET/HEAD, or if CSRF token is valid. + def verified_request? + Gitlab::RequestForgeryProtection.verified?(current_request.env) + end + + def ensure_action_dispatch_request(request) + return request if request.is_a?(ActionDispatch::Request) + + ActionDispatch::Request.new(request.env) + end + + def current_request + @current_request ||= ensure_action_dispatch_request(request) + end + end + end +end diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml new file mode 100644 index 00000000000..8242821cedc --- /dev/null +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -0,0 +1,52 @@ +# For background migrations we define a custom set of rules to make it less +# difficult to review these migrations. To reduce the complexity of these +# migrations some rules may be stricter than the defaults set in the root +# .rubocop.yml file. +--- +inherit_from: ../../../.rubocop.yml + +Metrics/AbcSize: + Enabled: true + Max: 30 + Details: > + Code that involves a lot of branches can be very hard to wrap your head + around. + +Metrics/PerceivedComplexity: + Enabled: true + +Metrics/LineLength: + Enabled: true + Details: > + Long lines are very hard to read and make it more difficult to review + changes. + +Metrics/MethodLength: + Enabled: true + Max: 30 + Details: > + Long methods can be very hard to review. Consider splitting this method up + into separate methods. + +Metrics/ClassLength: + Enabled: true + Details: > + Long classes can be very hard to review. Consider splitting this class up + into multiple classes. + +Metrics/BlockLength: + Enabled: true + Details: > + Long blocks can be hard to read. Consider splitting the code into separate + methods. + +Style/Documentation: + Enabled: true + Details: > + Adding documentation makes it easier to figure out what a migration is + supposed to do. + +Style/FrozenStringLiteralComment: + Enabled: true + Details: >- + This removes the need for calling "freeze", reducing noise in the code. diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index c88eb9783ed..03b17b319fa 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class CreateForkNetworkMembershipsRange @@ -51,10 +55,20 @@ module Gitlab FROM projects WHERE forked_project_links.forked_from_project_id = projects.id ) + AND NOT EXISTS ( + SELECT true + FROM forked_project_links AS parent_links + WHERE parent_links.forked_to_project_id = forked_project_links.forked_from_project_id + AND NOT EXISTS ( + SELECT true + FROM projects + WHERE parent_links.forked_from_project_id = projects.id + ) + ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} MISSING_MEMBERS - ForkNetworkMember.count_by_sql(count_sql) > 0 + ForkedProjectLink.count_by_sql(count_sql) > 0 end def log(message) diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb index e94719db72e..c2bf42f846d 100644 --- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys class GpgKey < ActiveRecord::Base self.table_name = 'gpg_keys' diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb index b1411be3016..a1af045a71f 100644 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class DeleteConflictingRedirectRoutesRange diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 380802258f5..fd5cbf76e47 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -1,3 +1,9 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class DeserializeMergeRequestDiffsAndCommits diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb index 91540127ea9..0a8a4313cd5 100644 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateBuildStageIdReference diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 432f7c3e706..84ac00f1a5c 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration # Class that migrates events for the new push event payloads setup. All diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index b1ff0900709..0e5c7f092f2 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateStageStatus diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb index 0881244ed49..7f243073fd0 100644 --- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MigrateSystemUploadsToNewFolder diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb index 07cec96bcc3..a4ef51fd0e8 100644 --- a/lib/gitlab/background_migration/move_personal_snippet_files.rb +++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class MovePersonalSnippetFiles diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb index bc53e6d7f94..85749366bfd 100644 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -1,3 +1,10 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/ClassLength +# rubocop:disable Metrics/BlockLength +# rubocop:disable Style/Documentation + module Gitlab module BackgroundMigration class NormalizeLdapExternUidsRange diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb index 2ef3a207dd3..a976cb4c243 100644 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -1,25 +1,55 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration + # This background migration is going to create all `fork_networks` and + # the `fork_network_members` for the roots of fork networks based on the + # existing `forked_project_links`. + # + # When the source of a fork is deleted, we will create the fork with the + # target project as the root. This way, when there are forks of the target + # project, they will be joined into the same fork network. + # + # When the `fork_networks` and memberships for the root projects are created + # the `CreateForkNetworkMembershipsRange` migration is scheduled. This + # migration will create the memberships for all remaining forks-of-forks class PopulateForkNetworksRange def perform(start_id, end_id) - log("Creating fork networks for forked project links: #{start_id} - #{end_id}") + create_fork_networks_for_existing_projects(start_id, end_id) + create_fork_networks_for_missing_projects(start_id, end_id) + create_fork_networks_memberships_for_root_projects(start_id, end_id) + + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength + BackgroundMigrationWorker.perform_in( + delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] + ) + end + def create_fork_networks_for_existing_projects(start_id, end_id) + log("Creating fork networks: #{start_id} - #{end_id}") ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS INSERT INTO fork_networks (root_project_id) SELECT DISTINCT forked_project_links.forked_from_project_id FROM forked_project_links + -- Exclude the forks that are not the first level fork of a project WHERE NOT EXISTS ( SELECT true FROM forked_project_links inner_links WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id ) + + /* Exclude the ones that are already created, in case the fork network + was already created for another fork of the project. + */ AND NOT EXISTS ( SELECT true FROM fork_networks WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id ) + + -- Only create a fork network for a root project that still exists AND EXISTS ( SELECT true FROM projects @@ -27,7 +57,45 @@ module Gitlab ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} INSERT_NETWORKS + end + + def create_fork_networks_for_missing_projects(start_id, end_id) + log("Creating fork networks with missing root: #{start_id} - #{end_id}") + ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS + INSERT INTO fork_networks (root_project_id) + SELECT DISTINCT forked_project_links.forked_to_project_id + + FROM forked_project_links + + -- Exclude forks that are not the root forks + WHERE NOT EXISTS ( + SELECT true + FROM forked_project_links inner_links + WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id + ) + + /* Exclude the ones that are already created, in case this migration is + re-run + */ + AND NOT EXISTS ( + SELECT true + FROM fork_networks + WHERE forked_project_links.forked_to_project_id = fork_networks.root_project_id + ) + + /* Exclude projects for which the project still exists, those are + Processed in the previous step of this migration + */ + AND NOT EXISTS ( + SELECT true + FROM projects + WHERE projects.id = forked_project_links.forked_from_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_NETWORKS + end + def create_fork_networks_memberships_for_root_projects(start_id, end_id) log("Creating memberships for root projects: #{start_id} - #{end_id}") ActiveRecord::Base.connection.execute <<~INSERT_ROOT @@ -36,8 +104,12 @@ module Gitlab FROM fork_networks + /* Joining both on forked_from- and forked_to- so we could create the + memberships for forks for which the source was deleted + */ INNER JOIN forked_project_links ON forked_project_links.forked_from_project_id = fork_networks.root_project_id + OR forked_project_links.forked_to_project_id = fork_networks.root_project_id WHERE NOT EXISTS ( SELECT true @@ -46,9 +118,6 @@ module Gitlab ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} INSERT_ROOT - - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY - BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]) end def log(message) diff --git a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb new file mode 100644 index 00000000000..dcac355e1b0 --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateMergeRequestsLatestMergeRequestDiffId + BATCH_SIZE = 1_000 + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include ::EachBatch + end + + def perform(start_id, stop_id) + update = ' + latest_merge_request_diff_id = ( + SELECT MAX(id) + FROM merge_request_diffs + WHERE merge_requests.id = merge_request_diffs.merge_request_id + )'.squish + + MergeRequest + .where(id: start_id..stop_id) + .where(latest_merge_request_diff_id: nil) + .each_batch(of: BATCH_SIZE) do |relation| + + relation.update_all(update) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb new file mode 100644 index 00000000000..81e95e5832d --- /dev/null +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -0,0 +1,259 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class processes a batch of rows in `untracked_files_for_uploads` by + # adding each file to the `uploads` table if it does not exist. + class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength + # This class is responsible for producing the attributes necessary to + # track an uploaded file in the `uploads` table. + class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength + self.table_name = 'untracked_files_for_uploads' + + # Ends with /:random_hex/:filename + FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} + FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}} + + # These regex patterns are tested against a relative path, relative to + # the upload directory. + # For convenience, if there exists a capture group in the pattern, then + # it indicates the model_id. + PATH_PATTERNS = [ + { + pattern: %r{\A-/system/appearance/logo/(\d+)/}, + uploader: 'AttachmentUploader', + model_type: 'Appearance' + }, + { + pattern: %r{\A-/system/appearance/header_logo/(\d+)/}, + uploader: 'AttachmentUploader', + model_type: 'Appearance' + }, + { + pattern: %r{\A-/system/note/attachment/(\d+)/}, + uploader: 'AttachmentUploader', + model_type: 'Note' + }, + { + pattern: %r{\A-/system/user/avatar/(\d+)/}, + uploader: 'AvatarUploader', + model_type: 'User' + }, + { + pattern: %r{\A-/system/group/avatar/(\d+)/}, + uploader: 'AvatarUploader', + model_type: 'Namespace' + }, + { + pattern: %r{\A-/system/project/avatar/(\d+)/}, + uploader: 'AvatarUploader', + model_type: 'Project' + }, + { + pattern: FILE_UPLOADER_PATH, + uploader: 'FileUploader', + model_type: 'Project' + } + ].freeze + + def to_h + @upload_hash ||= { + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size, + checksum: checksum + } + end + + def upload_path + # UntrackedFile#path is absolute, but Upload#path depends on uploader + @upload_path ||= + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH) + matchd[0].sub(%r{\A/}, '') # remove leading slash + else + path + end + end + + def uploader + matching_pattern_map[:uploader] + end + + def model_type + matching_pattern_map[:model_type] + end + + def model_id + return @model_id if defined?(@model_id) + + pattern = matching_pattern_map[:pattern] + matchd = path_relative_to_upload_dir.match(pattern) + + # If something is captured (matchd[1] is not nil), it is a model_id + # Only the FileUploader pattern will not match an ID + @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id + end + + def file_size + File.size(absolute_path) + end + + def checksum + Digest::SHA256.file(absolute_path).hexdigest + end + + private + + def matching_pattern_map + @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map| + path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + end + + unless @matching_pattern_map + raise "Unknown upload path pattern \"#{path}\"" + end + + @matching_pattern_map + end + + def file_uploader_model_id + matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE) + not_found_msg = <<~MSG + Could not capture project full_path from a FileUploader path: + "#{path_relative_to_upload_dir}" + MSG + raise not_found_msg unless matchd + + full_path = matchd[1] + project = Project.find_by_full_path(full_path) + return nil unless project + + project.id + end + + # Not including a leading slash + def path_relative_to_upload_dir + upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength + base = %r{\A#{Regexp.escape(upload_dir)}/} + @path_relative_to_upload_dir ||= path.sub(base, '') + end + + def absolute_path + File.join(CarrierWave.root, path) + end + end + + # This class is used to query the `uploads` table. + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + end + + def perform(start_id, end_id) + return unless migrate? + + files = UntrackedFile.where(id: start_id..end_id) + processed_files = insert_uploads_if_needed(files) + processed_files.delete_all + + drop_temp_table_if_finished + end + + private + + def migrate? + UntrackedFile.table_exists? && Upload.table_exists? + end + + def insert_uploads_if_needed(files) + filtered_files, error_files = filter_error_files(files) + filtered_files = filter_existing_uploads(filtered_files) + filtered_files = filter_deleted_models(filtered_files) + insert(filtered_files) + + processed_files = files.where.not(id: error_files.map(&:id)) + processed_files + end + + def filter_error_files(files) + files.partition do |file| + begin + file.to_h + true + rescue => e + msg = <<~MSG + Error parsing path "#{file.path}": + #{e.message} + #{e.backtrace.join("\n ")} + MSG + Rails.logger.error(msg) + false + end + end + end + + def filter_existing_uploads(files) + paths = files.map(&:upload_path) + existing_paths = Upload.where(path: paths).pluck(:path).to_set + + files.reject do |file| + existing_paths.include?(file.upload_path) + end + end + + # There are files on disk that are not in the uploads table because their + # model was deleted, and we don't delete the files on disk. + def filter_deleted_models(files) + ids = deleted_model_ids(files) + + files.reject do |file| + ids[file.model_type].include?(file.model_id) + end + end + + def deleted_model_ids(files) + ids = { + 'Appearance' => [], + 'Namespace' => [], + 'Note' => [], + 'Project' => [], + 'User' => [] + } + + # group model IDs by model type + files.each do |file| + ids[file.model_type] << file.model_id + end + + ids.each do |model_type, model_ids| + model_class = Object.const_get(model_type) + found_ids = model_class.where(id: model_ids.uniq).pluck(:id) + deleted_ids = ids[model_type] - found_ids + ids[model_type] = deleted_ids + end + + ids + end + + def insert(files) + rows = files.map do |file| + file.to_h.merge(created_at: 'NOW()') + end + + Gitlab::Database.bulk_insert('uploads', + rows, + disable_quote: :created_at) + end + + def drop_temp_table_if_finished + if UntrackedFile.all.empty? + UntrackedFile.connection.drop_table(:untracked_files_for_uploads, + if_exists: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb new file mode 100644 index 00000000000..476c46341ae --- /dev/null +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class finds all non-hashed uploaded file paths and saves them to a + # `untracked_files_for_uploads` table. + class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength + # For bulk_queue_background_migration_jobs_by_range + include Database::MigrationHelpers + + FIND_BATCH_SIZE = 500 + RELATIVE_UPLOAD_DIR = "uploads".freeze + ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/} + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + + # This class is used to iterate over batches of + # `untracked_files_for_uploads` rows. + class UntrackedFile < ActiveRecord::Base + include EachBatch + + self.table_name = 'untracked_files_for_uploads' + end + + def perform + ensure_temporary_tracking_table_exists + + # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since + # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be + # slow, start with an empty table for Postgres < 9.5. + # That way we can do bulk inserts at ~30x the speed of individual + # inserts (~20 minutes worth of inserts at GitLab.com scale instead of + # ~10 hours). + # In all other cases, installations will get both bulk inserts and the + # ability for these jobs to retry without having to clear and reinsert. + clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates? + + store_untracked_file_paths + + schedule_populate_untracked_uploads_jobs + end + + private + + def ensure_temporary_tracking_table_exists + table_name = :untracked_files_for_uploads + unless UntrackedFile.connection.table_exists?(table_name) + UntrackedFile.connection.create_table table_name do |t| + t.string :path, limit: 600, null: false + t.index :path, unique: true + end + end + end + + def clear_untracked_file_paths + UntrackedFile.delete_all + end + + def store_untracked_file_paths + return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) + + each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths| + insert_file_paths(file_paths) + end + end + + def each_file_batch(search_dir, batch_size, &block) + cmd = build_find_command(search_dir) + + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + yield_paths_in_batches(stdout, batch_size, &block) + + raise "Find command failed" unless status_thread.value.success? + end + end + + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '') + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) + end + + def build_find_command(search_dir) + cmd = %W[find -L #{search_dir} + -type f + ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) + -print0] + + ionice = which_ionice + cmd = %W[#{ionice} -c Idle] + cmd if ionice + + log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + Rails.logger.info log_msg + + cmd + end + + def which_ionice + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, + # even though it isn't very nice + false + end + + def insert_file_paths(file_paths) + sql = insert_sql(file_paths) + + ActiveRecord::Base.connection.execute(sql) + end + + def insert_sql(file_paths) + if postgresql_pre_9_5? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" + elsif postgresql? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ + " ON CONFLICT DO NOTHING;" + else # MySQL + "INSERT IGNORE INTO"\ + " #{table_columns_and_values_for_insert(file_paths)};" + end + end + + def table_columns_and_values_for_insert(file_paths) + values = file_paths.map do |file_path| + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength + end.join(', ') + + "#{UntrackedFile.table_name} (path) VALUES #{values}" + end + + def postgresql? + @postgresql ||= Gitlab::Database.postgresql? + end + + def can_bulk_insert_and_ignore_duplicates? + !postgresql_pre_9_5? + end + + def postgresql_pre_9_5? + @postgresql_pre_9_5 ||= postgresql? && + Gitlab::Database.version.to_f < 9.5 + end + + def schedule_populate_untracked_uploads_jobs + bulk_queue_background_migration_jobs_by_range( + UntrackedFile, FOLLOW_UP_MIGRATION) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 033ecd15749..d48ae17aeaf 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -61,9 +61,9 @@ module Gitlab def import_wiki return if project.wiki.repository_exists? - path_with_namespace = "#{project.full_path}.wiki" + disk_path = project.wiki.disk_path import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) + gitlab_shell.import_repository(project.repository_storage_path, disk_path, import_url) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index 5b32fca00a4..9c9e6668e6f 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -16,6 +16,7 @@ module Gitlab @changes ||= begin @raw_changes.map do |change| next if change.blank? + oldrev, newrev, ref = change.strip.split(' ') { oldrev: oldrev, newrev: newrev, ref: ref } end.compact diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index 27a95764dc1..f7276a380dc 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -15,7 +15,10 @@ module Gitlab return false unless new_lfs_pointers.present? - existing_count = @project.lfs_objects.where(oid: new_lfs_pointers.map(&:lfs_oid)).count + existing_count = @project.lfs_storage_project + .lfs_objects + .where(oid: new_lfs_pointers.map(&:lfs_oid)) + .count existing_count != new_lfs_pointers.count end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index a788fb3fcbc..0bbd60d8ffe 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -98,6 +98,7 @@ module Gitlab def read_string(gz) string_size = read_uint32(gz) return nil unless string_size + gz.read(string_size) end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 22941d48edf..5b2f09e03ea 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -43,6 +43,7 @@ module Gitlab def parent return nil unless has_parent? + self.class.new(@path.to_s.chomp(basename), @entries) end @@ -64,6 +65,7 @@ module Gitlab def directories(opts = {}) return [] unless directory? + dirs = children.select(&:directory?) return dirs unless has_parent? && opts[:parent] @@ -74,6 +76,7 @@ module Gitlab def files return [] unless directory? + children.select(&:file?) end diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index b88b2e36d53..c811f88f483 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -8,6 +8,7 @@ module Gitlab def from_image(job) image = Gitlab::Ci::Build::Image.new(job.options[:image]) return unless image.valid? + image end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 6555c589173..2844be80a84 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -37,6 +37,7 @@ module Gitlab def value return { name: @config } if string? return @config if hash? + {} end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index 0159179f0a9..eb606b57667 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -111,6 +111,7 @@ module Gitlab def validate_string_or_regexp(value) return false unless value.is_a?(String) return validate_regexp(value) if look_like_regexp?(value) + true end end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb new file mode 100644 index 00000000000..a126dded1ae --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -0,0 +1,58 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Build < Chain::Base + include Chain::Helpers + + def perform! + @pipeline.assign_attributes( + source: @command.source, + project: @project, + ref: ref, + sha: sha, + before_sha: before_sha, + tag: tag_exists?, + trigger_requests: Array(@command.trigger_request), + user: @current_user, + pipeline_schedule: @command.schedule, + protected: protected_ref? + ) + + @pipeline.set_config_source + end + + def break? + false + end + + private + + def ref + @ref ||= Gitlab::Git.ref_name(origin_ref) + end + + def sha + @project.commit(origin_sha || origin_ref).try(:id) + end + + def origin_ref + @command.origin_ref + end + + def origin_sha + @command.checkout_sha || @command.after_sha + end + + def before_sha + @command.checkout_sha || @command.before_sha || Gitlab::Git::BLANK_SHA + end + + def protected_ref? + @project.protected_for?(ref) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d5e17a123df..d19a2519803 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -17,11 +17,27 @@ module Gitlab end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") + ensure + if pipeline.builds.where(stage_id: nil).any? + invalid_builds_counter.increment(node: hostname) + end end def break? !pipeline.persisted? end + + private + + def invalid_builds_counter + @counter ||= Gitlab::Metrics + .counter(:gitlab_ci_invalid_builds_total, + 'Invalid builds without stage assigned counter') + end + + def hostname + @hostname ||= Socket.gethostname + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 015f2988327..e24630656d3 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -5,20 +5,19 @@ module Gitlab class Sequence def initialize(pipeline, command, sequence) @pipeline = pipeline + @command = command + @sequence = sequence @completed = [] - - @sequence = sequence.map do |chain| - chain.new(pipeline, command) - end end def build! - @sequence.each do |step| - step.perform! + @sequence.each do |chain| + step = chain.new(@pipeline, @command) + step.perform! break if step.break? - @completed << step + @completed.push(step) end @pipeline.tap do diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 2479b4a7706..9230894877f 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -3,7 +3,6 @@ module Gitlab class PlanEventFetcher < BaseEventFetcher def initialize(*args) @projections = [mr_diff_table[:id], - mr_diff_table[:st_commits], issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) @@ -37,12 +36,7 @@ module Gitlab def first_time_reference_commit(event) return nil unless event && merge_request_diff_commits - commits = - if event['st_commits'].present? - YAML.load(event['st_commits']) - else - merge_request_diff_commits[event['id'].to_i] - end + commits = merge_request_diff_commits[event['id'].to_i] return nil if commits.blank? diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index f07fd1dfdda..633de9f9776 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -2,6 +2,7 @@ module Gitlab class Daemon def self.initialize_instance(*args) raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) Kernel.at_exit(&@instance.method(:stop)) @instance diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 43a00d6cedb..e51794fef99 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -50,6 +50,10 @@ module Gitlab postgresql? && version.to_f >= 9.3 end + def self.replication_slots_supported? + postgresql? && version.to_f >= 9.4 + end + def self.nulls_last_order(field, direction = 'ASC') order = "#{field} #{direction}" @@ -108,20 +112,47 @@ module Gitlab end end - def self.bulk_insert(table, rows) + # Bulk inserts a number of rows into a table, optionally returning their + # IDs. + # + # table - The name of the table to insert the rows into. + # rows - An Array of Hash instances, each mapping the columns to their + # values. + # return_ids - When set to true the return value will be an Array of IDs of + # the inserted rows, this only works on PostgreSQL. + # disable_quote - A key or an Array of keys to exclude from quoting (You + # become responsible for protection from SQL injection for + # these keys!) + def self.bulk_insert(table, rows, return_ids: false, disable_quote: []) return if rows.empty? keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } + return_ids = false if mysql? + disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| - row.values_at(*keys).map { |value| connection.quote(value) } + keys.map do |k| + disable_quote.include?(k) ? row[k] : connection.quote(row[k]) + end end - connection.execute <<-EOF + sql = <<-EOF INSERT INTO #{table} (#{columns.join(', ')}) VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} EOF + + if return_ids + sql << 'RETURNING id' + end + + result = connection.execute(sql) + + if return_ids + result.values.map { |tuple| tuple[0].to_i } + else + [] + end end def self.sanitize_timestamp(timestamp) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 2c35da8f1aa..3f65bc912de 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -220,6 +220,15 @@ module Gitlab # column - The name of the column to update. # value - The value for the column. # + # The `value` argument is typically a literal. To perform a computed + # update, an Arel literal can be used instead: + # + # update_value = Arel.sql('bar * baz') + # + # update_column_in_batches(:projects, :foo, update_value) do |table, query| + # query.where(table[:some_column].eq('hello')) + # end + # # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop # determines this method to be too complex while there's no way to make it # less "complex" without introducing extra methods (which actually will @@ -694,14 +703,14 @@ into similar problems in the future (e.g. when new tables are created). # We push multiple jobs at a time to reduce the time spent in # Sidekiq/Redis operations. We're using this buffer based approach so we # don't need to run additional queries for every range. - BackgroundMigrationWorker.perform_bulk(jobs) + BackgroundMigrationWorker.bulk_perform_async(jobs) jobs.clear end jobs << [job_class_name, [start_id, end_id]] end - BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty? + BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty? end # Queues background migration jobs for an entire table, batched by ID range. diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index 5481024db8e..7e492938eac 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -68,6 +68,11 @@ module Gitlab has_one :route, as: :source self.table_name = 'projects' + HASHED_STORAGE_FEATURES = { + repository: 1, + attachments: 2 + }.freeze + def repository_storage_path Gitlab.config.repositories.storages[repository_storage]['path'] end @@ -76,6 +81,13 @@ module Gitlab def self.name 'Project' end + + def hashed_storage?(feature) + raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature) + return false unless respond_to?(:storage_version) + + self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature] + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 75a75f61953..d32616862f0 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -22,9 +22,11 @@ module Gitlab end def move_project_folders(project, old_full_path, new_full_path) - move_repository(project, old_full_path, new_full_path) - move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") - move_uploads(old_full_path, new_full_path) + unless project.hashed_storage?(:repository) + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + end + move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments) move_pages(old_full_path, new_full_path) end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index c98eefbce25..88e0db830f6 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -13,9 +13,9 @@ module Gitlab def ==(other) other.is_a?(self.class) && - shas_equal?(base_sha, other.base_sha) && - shas_equal?(start_sha, other.start_sha) && - shas_equal?(head_sha, other.head_sha) + Git.shas_eql?(base_sha, other.base_sha) && + Git.shas_eql?(start_sha, other.start_sha) && + Git.shas_eql?(head_sha, other.head_sha) end alias_method :eql?, :== @@ -47,22 +47,6 @@ module Gitlab CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) end end - - private - - def shas_equal?(sha1, sha2) - return true if sha1 == sha2 - return false if sha1.nil? || sha2.nil? - return false unless sha1.class == sha2.class - - length = [sha1.length, sha2.length].min - - # If either of the shas is below the minimum length, we cannot be sure - # that they actually refer to the same commit because of hash collision. - return false if length < Commit::MIN_SHA_LENGTH - - sha1[0, length] == sha2[0, length] - end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index ea5891a028a..d0cfe2386ca 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -25,6 +25,10 @@ module Gitlab @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + + # Ensure items are collected in the the batch + new_blob + old_blob end def position(position_marker, position_type: :text) @@ -95,21 +99,15 @@ module Gitlab end def new_blob - return @new_blob if defined?(@new_blob) - - sha = new_content_sha - return @new_blob = nil unless sha + return unless new_content_sha - @new_blob = repository.blob_at(sha, file_path) + Blob.lazy(repository.project, new_content_sha, file_path) end def old_blob - return @old_blob if defined?(@old_blob) - - sha = old_content_sha - return @old_blob = nil unless sha + return unless old_content_sha - @old_blob = repository.blob_at(sha, old_path) + Blob.lazy(repository.project, old_content_sha, old_path) end def content_sha diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 88ae65cb468..a6007ebf531 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -22,10 +22,7 @@ module Gitlab end def diff_files - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37445 - Gitlab::GitalyClient.allow_n_plus_1_calls do - @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } - end + @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } end def diff_file_with_old_path(old_path) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 55708d42161..54783a07919 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -70,7 +70,7 @@ module Gitlab def find_changed_line_pairs(lines) # Prefixes of all diff lines, indicating their types # For example: `" - + -+ ---+++ --+ -++"` - line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ') + line_prefixes = lines.each_with_object("") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') changed_line_pairs = [] line_prefixes.scan(LINE_PAIRS_PATTERN) do @@ -102,6 +102,7 @@ module Gitlab new_char = b[pos] break if old_char != new_char + length += 1 end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7dc9cc7c281..8302f30a0a2 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -30,6 +30,7 @@ module Gitlab line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 next if line_old <= 1 && line_new <= 1 # top of file + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index ccfb908bcca..690b27cde81 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -125,6 +125,7 @@ module Gitlab def find_diff_file(repository) return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) + comparison.diffs(paths: paths, expanded: true).diff_files.first end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 0ea534a5fd0..4a9d3e52fae 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -31,16 +31,22 @@ module Gitlab def check ensure_patches_dir - generate_patch(ce_branch, ce_patch_full_path) + add_remote('canonical-ce', "#{DEFAULT_CE_PROJECT_URL}.git") + generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, remote: 'canonical-ce') ensure_ee_repo Dir.chdir(ee_repo_dir) do step("In the #{ee_repo_dir} directory") + add_remote('canonical-ee', EE_REPO_URL) + status = catch(:halt_check) do ce_branch_compat_check! delete_ee_branches_locally! ee_branch_presence_check! + + step("Checking out #{ee_branch_found}", %W[git checkout -b #{ee_branch_found} canonical-ee/#{ee_branch_found}]) + generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, remote: 'canonical-ee') ee_branch_compat_check! end @@ -56,6 +62,13 @@ module Gitlab private + def add_remote(name, url) + step( + "Adding the #{name} remote (#{url})", + %W[git remote add #{name} #{url}] + ) + end + def ensure_ee_repo if Dir.exist?(ee_repo_dir) step("#{ee_repo_dir} already exists") @@ -71,14 +84,14 @@ module Gitlab FileUtils.mkdir_p(patches_dir) end - def generate_patch(branch, patch_path) + def generate_patch(branch:, patch_path:, remote:) FileUtils.rm(patch_path, force: true) - find_merge_base_with_master(branch: branch) + find_merge_base_with_master(branch: branch, master_remote: remote) step( - "Generating the patch against origin/master in #{patch_path}", - %w[git diff --binary origin/master...HEAD] + "Generating the patch against #{remote}/master in #{patch_path}", + %W[git diff --binary #{remote}/master...origin/#{branch}] ) do |output, status| throw(:halt_check, :ko) unless status.zero? @@ -96,14 +109,14 @@ module Gitlab end def ee_branch_presence_check! - _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch origin #{ee_branch_prefix}]) + _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch canonical-ee #{ee_branch_prefix}]) if status.zero? @ee_branch_found = ee_branch_prefix return end - _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}]) + _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch canonical-ee #{ee_branch_suffix}]) if status.zero? @ee_branch_found = ee_branch_suffix @@ -116,10 +129,6 @@ module Gitlab end def ee_branch_compat_check! - step("Checking out origin/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} FETCH_HEAD]) - - generate_patch(ee_branch_found, ee_patch_full_path) - unless check_patch(ee_patch_full_path).zero? puts puts ee_branch_doesnt_apply_cleanly_msg @@ -133,8 +142,7 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) - step("Resetting to latest master", %w[git reset --hard origin/master]) - step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo_url} #{ce_branch}]) + step("Resetting to latest master", %w[git reset --hard canonical-ee/master]) step( "Checking if #{patch_path} applies cleanly to EE/master", # Don't use --check here because it can result in a 0-exit status even @@ -171,10 +179,10 @@ module Gitlab command(%W[git branch --delete --force #{ee_branch_suffix}]) end - def merge_base_found? + def merge_base_found?(master_remote:, branch:) step( - "Finding merge base with master", - %w[git merge-base origin/master HEAD] + "Finding merge base with #{master_remote}/master", + %W[git merge-base #{master_remote}/master origin/#{branch}] ) do |output, status| if status.zero? puts "Merge base was found: #{output}" @@ -183,7 +191,7 @@ module Gitlab end end - def find_merge_base_with_master(branch:) + def find_merge_base_with_master(branch:, master_remote:) # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 @@ -192,19 +200,19 @@ module Gitlab depth += Math.exp(factor).to_i # Repository is initially cloned with a depth of 20 so we need to fetch # deeper in the case the branch has more than 20 commits on top of master - fetch(branch: branch, depth: depth) - fetch(branch: 'master', depth: depth) + fetch(branch: branch, depth: depth, remote: 'origin') + fetch(branch: 'master', depth: depth, remote: master_remote) - merge_base_found? + merge_base_found?(master_remote: master_remote, branch: branch) end - raise "\n#{branch} is too far behind master, please rebase it!\n" unless success + raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success end - def fetch(branch:, depth:) + def fetch(branch:, depth:, remote: 'origin') step( "Fetching deeper...", - %W[git fetch --depth=#{depth} --prune origin +refs/heads/#{branch}:refs/remotes/origin/#{branch}] + %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/#{remote}/#{branch}] ) do |output, status| raise "Fetch failed: #{output}" unless status.zero? end @@ -304,8 +312,8 @@ module Gitlab 1. Create a new branch from master and cherry-pick your CE commits # In the EE repo - $ git fetch origin - $ git checkout -b #{ee_branch_prefix} origin/master + $ git fetch #{EE_REPO_URL} master + $ git checkout -b #{ee_branch_prefix} FETCH_HEAD $ git fetch #{ce_repo_url} #{ce_branch} $ git cherry-pick SHA # Repeat for all the commits you want to pick @@ -314,10 +322,9 @@ module Gitlab 2. Apply your branch's patch to EE # In the EE repo - $ git fetch origin master - $ git checkout -b #{ee_branch_prefix} origin/master - $ wget #{patch_url} - $ git apply --3way #{ce_patch_name} + $ git fetch #{EE_REPO_URL} master + $ git checkout -b #{ee_branch_prefix} FETCH_HEAD + $ wget #{patch_url} && git apply --3way #{ce_patch_name} At this point you might have conflicts such as: diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index b07c68d1498..e08b5be8984 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,3 +1,4 @@ +require 'gitlab/email/handler/create_merge_request_handler' require 'gitlab/email/handler/create_note_handler' require 'gitlab/email/handler/create_issue_handler' require 'gitlab/email/handler/unsubscribe_handler' @@ -8,6 +9,7 @@ module Gitlab HANDLERS = [ UnsubscribeHandler, CreateNoteHandler, + CreateMergeRequestHandler, CreateIssueHandler ].freeze diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb new file mode 100644 index 00000000000..c63666b98c1 --- /dev/null +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -0,0 +1,67 @@ +require 'gitlab/email/handler/base_handler' +require 'gitlab/email/handler/reply_processing' + +module Gitlab + module Email + module Handler + class CreateMergeRequestHandler < BaseHandler + include ReplyProcessing + attr_reader :project_path, :incoming_email_token + + def initialize(mail, mail_key) + super(mail, mail_key) + if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) + @project_path, @incoming_email_token = m.captures + end + end + + def can_handle? + @project_path && @incoming_email_token + end + + def execute + raise ProjectNotFound unless project + + validate_permission!(:create_merge_request) + + verify_record!( + record: create_merge_request, + invalid_exception: InvalidMergeRequestError, + record_name: 'merge_request') + end + + def author + @author ||= User.find_by(incoming_email_token: incoming_email_token) + end + + def project + @project ||= Project.find_by_full_path(project_path) + end + + def metrics_params + super.merge(project: project&.full_path) + end + + private + + def create_merge_request + merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute + + if merge_request.errors.any? + merge_request + else + MergeRequests::CreateService.new(project, author).create(merge_request) + end + end + + def merge_request_params + { + source_project_id: project.id, + source_branch: mail.subject, + target_project_id: project.id + } + end + end + end + end +end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 5894384da5d..ea80e21532e 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -16,6 +16,7 @@ module Gitlab noteable = sent_notification.noteable raise NoteableNotFoundError unless noteable + noteable.unsubscribe(sent_notification.recipient) end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index c8f4591d060..d8c594ad0e7 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -13,8 +13,10 @@ module Gitlab UserBlockedError = Class.new(ProcessingError) UserNotAuthorizedError = Class.new(ProcessingError) NoteableNotFoundError = Class.new(ProcessingError) - InvalidNoteError = Class.new(ProcessingError) - InvalidIssueError = Class.new(ProcessingError) + InvalidRecordError = Class.new(ProcessingError) + InvalidNoteError = Class.new(InvalidRecordError) + InvalidIssueError = Class.new(InvalidRecordError) + InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) class Receiver diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 99dfee3dd9b..582028493e9 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -17,6 +17,10 @@ module Gitlab return nil unless message.respond_to?(:force_encoding) return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + if message.respond_to?(:frozen?) && message.frozen? + message = message.dup + end + message.force_encoding("UTF-8") return message if message.valid_encoding? diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb index 2152182b37f..acb000e3e23 100644 --- a/lib/gitlab/fogbugz_import/client.rb +++ b/lib/gitlab/fogbugz_import/client.rb @@ -45,6 +45,7 @@ module Gitlab project_name = repo(project_id).name res = @api.command(:search, q: "project:'#{project_name}'", cols: 'ixPersonAssignedTo,ixPersonOpenedBy,ixPersonClosedBy,sStatus,sPriority,sCategory,fOpen,sTitle,sLatestTextSummary,dtOpened,dtClosed,dtResolved,dtLastUpdated,events') return [] unless res['cases']['count'].to_i > 0 + res['cases']['case'] end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 3dcee681c72..5e426b13ade 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -18,6 +18,7 @@ module Gitlab def execute return true unless repo.valid? + client = Gitlab::FogbugzImport::Client.new(token: fb_session[:token], uri: fb_session[:uri]) @cases = client.cases(@repo.id.to_i) @@ -206,6 +207,7 @@ module Gitlab def format_content(raw_content) return raw_content if raw_content.nil? + linkify_issues(escape_for_markdown(raw_content)) end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 1f31cdbc96d..1f7c35cafaa 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -70,6 +70,18 @@ module Gitlab def diff_line_code(file_path, new_line_position, old_line_position) "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" end + + def shas_eql?(sha1, sha2) + return false if sha1.nil? || sha2.nil? + return false unless sha1.class == sha2.class + + # If either of the shas is below the minimum length, we cannot be sure + # that they actually refer to the same commit because of hash collision. + length = [sha1.length, sha2.length].min + return false if length < Gitlab::Git::Commit::MIN_SHA_LENGTH + + sha1[0, length] == sha2[0, length] + end end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index cc6c7609ec7..228d97a87ab 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -49,6 +49,7 @@ module Gitlab # Keep in mind that this method may allocate a lot of memory. It is up # to the caller to limit the number of blobs and blob_size_limit. # + # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 def batch(repository, blob_references, blob_size_limit: nil) blob_size_limit ||= MAX_DATA_DISPLAY_SIZE blob_references.map do |sha, path| @@ -102,6 +103,7 @@ module Gitlab if path_arr.size > 1 return nil unless entry[:type] == :tree + path_arr.shift find_entry_by_path(repository, entry[:oid], path_arr.join('/')) else @@ -178,6 +180,8 @@ module Gitlab ) end end + rescue Rugged::ReferenceError + nil end def rugged_raw(repository, sha, limit:) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index d5518814483..e90b158fb34 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -6,6 +6,7 @@ module Gitlab attr_accessor :raw_commit, :head + MIN_SHA_LENGTH = 7 SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, @@ -213,11 +214,17 @@ module Gitlab end def shas_with_signatures(repository, shas) - shas.select do |sha| - begin - Rugged::Commit.extract_signature(repository.rugged, sha) - rescue Rugged::OdbError - false + GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) + else + shas.select do |sha| + begin + Rugged::Commit.extract_signature(repository.rugged, sha) + rescue Rugged::OdbError + false + end + end end end end @@ -418,6 +425,20 @@ module Gitlab parent_ids.size > 1 end + def to_gitaly_commit + return raw_commit if raw_commit.is_a?(Gitaly::GitCommit) + + message_split = raw_commit.message.split("\n", 2) + Gitaly::GitCommit.new( + id: raw_commit.oid, + subject: message_split[0] ? message_split[0].chomp.b : "", + body: raw_commit.message.b, + parent_ids: raw_commit.parent_ids, + author: gitaly_commit_author_from_rugged(raw_commit.author), + committer: gitaly_commit_author_from_rugged(raw_commit.committer) + ) + end + private def init_from_hash(hash) @@ -463,6 +484,14 @@ module Gitlab def serialize_keys SERIALIZE_KEYS end + + def gitaly_commit_author_from_rugged(author_or_committer) + Gitaly::CommitAuthor.new( + name: author_or_committer[:name].b, + email: author_or_committer[:email].b, + date: Google::Protobuf::Timestamp.new(seconds: author_or_committer[:time].to_i) + ) + end end end end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index fc1595f1faf..b2a625e08fa 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,7 +2,7 @@ module Gitlab module Git module Conflict class File - attr_reader :content, :their_path, :our_path, :our_mode, :repository + attr_reader :content, :their_path, :our_path, :our_mode, :repository, :commit_oid def initialize(repository, commit_oid, conflict, content) @repository = repository diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index df509c5f4ce..de8cce41b6d 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -75,7 +75,7 @@ module Gitlab resolved_lines = file.resolve_lines(params[:sections]) new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") - new_file << "\n" if file.our_blob.data.ends_with?("\n") + new_file << "\n" if file.our_blob.data.end_with?("\n") elsif params[:content] new_file = file.resolve_content(params[:content]) end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ab94ba8a73a..7e8fe173056 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -72,7 +72,7 @@ module Gitlab # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, # it would be created from `start_branch_name`. - # If `start_project` is passed, and the branch doesn't exist, + # If `start_repository` is passed, and the branch doesn't exist, # it would try to find the commits from it instead of current repository. def with_branch( branch_name, @@ -80,15 +80,13 @@ module Gitlab start_repository: repository, &block) - # Refactoring aid - unless start_repository.is_a?(Gitlab::Git::Repository) - raise "expected a Gitlab::Git::Repository, got #{start_repository}" - end + Gitlab::Git.check_namespace!(start_repository) + start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - start_branch_name = nil if start_repository.empty_repo? + start_branch_name = nil if start_repository.empty? if start_branch_name && !start_repository.branch_exists?(start_branch_name) - raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.full_path}" + raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" end update_branch_with_hooks(branch_name) do diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb new file mode 100644 index 00000000000..6bd6e58feeb --- /dev/null +++ b/lib/gitlab/git/remote_repository.rb @@ -0,0 +1,84 @@ +module Gitlab + module Git + # + # When a Gitaly call involves two repositories instead of one we cannot + # assume that both repositories are on the same Gitaly server. In this + # case we need to make a distinction between the repository that the + # call is being made on (a Repository instance), and the "other" + # repository (a RemoteRepository instance). This is the reason why we + # have the RemoteRepository class in Gitlab::Git. + # + # When you make changes, be aware that gitaly-ruby sub-classes this + # class. + # + class RemoteRepository + attr_reader :path, :relative_path, :gitaly_repository + + def initialize(repository) + @relative_path = repository.relative_path + @gitaly_repository = repository.gitaly_repository + + # These instance variables will not be available in gitaly-ruby, where + # we have no disk access to this repository. + @repository = repository + @path = repository.path + end + + def empty? + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + # + # Caches and memoization used on the Rails side + !@repository.exists? || @repository.empty? + end + + def commit_id(revision) + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + @repository.commit(revision)&.sha + end + + def branch_exists?(name) + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + @repository.branch_exists?(name) + end + + # Compares self to a Gitlab::Git::Repository. This implementation uses + # 'self.gitaly_repository' so that it will also work in the + # GitalyRemoteRepository subclass defined in gitaly-ruby. + def same_repository?(other_repository) + gitaly_repository.storage_name == other_repository.storage && + gitaly_repository.relative_path == other_repository.relative_path + end + + def fetch_env + gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) + gitaly_address = gitaly_client.address(storage) + gitaly_token = gitaly_client.token(storage) + + request = Gitaly::SSHUploadPackRequest.new(repository: gitaly_repository) + env = { + 'GITALY_ADDRESS' => gitaly_address, + 'GITALY_PAYLOAD' => request.to_json, + 'GITALY_WD' => Dir.pwd, + 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" + } + env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? + + env + end + + private + + # Must return an object that responds to 'address' and 'storage'. + def gitaly_client + Gitlab::GitalyClient + end + + def storage + gitaly_repository.storage_name + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index df4ad586e12..73889328f36 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -18,6 +18,8 @@ module Gitlab GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze SEARCH_CONTEXT_LINES = 3 + REBASE_WORKTREE_PREFIX = 'rebase'.freeze + SQUASH_WORKTREE_PREFIX = 'squash'.freeze NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -58,7 +60,7 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver + attr_reader :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -66,7 +68,6 @@ module Gitlab @storage = storage @relative_path = relative_path @gl_repository = gl_repository - @gitaly_resolver = Gitlab::GitalyClient storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -74,9 +75,6 @@ module Gitlab @attributes = Gitlab::Git::Attributes.new(path) end - delegate :empty?, - to: :rugged - def ==(other) path == other.path end @@ -105,7 +103,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_repository_client.exists? else @@ -205,6 +203,13 @@ module Gitlab end end + # Git repository can contains some hidden refs like: + # /refs/notes/* + # /refs/git-as-svn/* + # /refs/pulls/* + # This refs by default not visible in project page and not cloned to client side. + alias_method :has_visible_content?, :has_local_branches? + def has_local_branches_rugged? rugged.branches.each(:local).any? do |ref| begin @@ -305,7 +310,13 @@ module Gitlab end def delete_all_refs_except(prefixes) - delete_refs(*all_ref_names_except(prefixes)) + gitaly_migrate(:ref_delete_refs) do |is_enabled| + if is_enabled + gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) + else + delete_refs(*all_ref_names_except(prefixes)) + end + end end # Returns an Array of all ref names, except when it's matching pattern @@ -500,7 +511,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - Commit.between(self, from, to).size + count_commits(ref: "#{from}..#{to}") end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -769,24 +780,21 @@ module Gitlab end def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - revert_tree_id = check_revert_content(commit, start_commit.sha) - raise CreateTreeError unless revert_tree_id - - committer = user_to_committer(user) + gitaly_migrate(:revert) do |is_enabled| + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - create_commit(message: message, - author: committer, - committer: committer, - tree: revert_tree_id, - parents: [start_commit.sha]) + if is_enabled + gitaly_operations_client.user_revert(args) + else + rugged_revert(args) + end end end @@ -804,44 +812,24 @@ module Gitlab end def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) - raise CreateTreeError unless cherry_pick_tree_id - - committer = user_to_committer(user) + gitaly_migrate(:cherry_pick) do |is_enabled| + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - create_commit(message: message, - author: { - email: commit.author_email, - name: commit.author_name, - time: commit.authored_date - }, - committer: committer, - tree: cherry_pick_tree_id, - parents: [start_commit.sha]) + if is_enabled + gitaly_operations_client.user_cherry_pick(args) + else + rugged_cherry_pick(args) + end end end - def check_cherry_pick_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << 1 if target_commit.merge_commit? - - cherry_pick_index = rugged.cherrypick_commit(*args) - return false if cherry_pick_index.conflicts? - - tree_id = cherry_pick_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - def diff_exists?(sha1, sha2) rugged.diff(sha1, sha2).size > 0 end @@ -920,6 +908,11 @@ module Gitlab false end + # Returns true if a remote exists. + def remote_exists?(name) + rugged.remotes[name].present? + end + # Update the specified remote using the values in the +options+ hash # # Example @@ -980,6 +973,10 @@ module Gitlab @attributes.attributes(path) end + def gitattribute(path, name) + attributes(path)[name] + end + def languages(ref = nil) Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| if is_enabled @@ -1009,23 +1006,22 @@ module Gitlab def with_repo_branch_commit(start_repository, start_branch_name) Gitlab::Git.check_namespace!(start_repository) + start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - return yield nil if start_repository.empty_repo? + return yield nil if start_repository.empty? - if start_repository == self + if start_repository.same_repository?(self) yield commit(start_branch_name) else - start_commit = start_repository.commit(start_branch_name) - - return yield nil unless start_commit + start_commit_id = start_repository.commit_id(start_branch_name) - sha = start_commit.sha + return yield nil unless start_commit_id - if branch_commit = commit(sha) + if branch_commit = commit(start_commit_id) yield branch_commit else with_repo_tmp_commit( - start_repository, start_branch_name, sha) do |tmp_commit| + start_repository, start_branch_name, start_commit_id) do |tmp_commit| yield tmp_commit end end @@ -1033,9 +1029,15 @@ module Gitlab end def with_repo_tmp_commit(start_repository, start_branch_name, sha) + source_ref = start_branch_name + + unless Gitlab::Git.branch_ref?(source_ref) + source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" + end + tmp_ref = fetch_ref( start_repository, - source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + source_ref: source_ref, target_ref: "refs/tmp/#{SecureRandom.hex}" ) @@ -1045,12 +1047,11 @@ module Gitlab end def fetch_source_branch!(source_repository, source_branch, local_ref) - with_repo_branch_commit(source_repository, source_branch) do |commit| - if commit - write_ref(local_ref, commit.sha) - true + Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled| + if is_enabled + gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) else - false + rugged_fetch_source_branch(source_repository, source_branch, local_ref) end end end @@ -1072,16 +1073,14 @@ module Gitlab raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] input = "update #{ref_path}\x00#{ref}\x00\x00" - output, status = circuit_breaker.perform do - popen(command, path) { |stdin| stdin.write(input) } - end - - raise GitError, output unless status.zero? + run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } end def fetch_ref(source_repository, source_ref:, target_ref:) + Gitlab::Git.check_namespace!(source_repository) + source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) + message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| if is_enabled gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) @@ -1097,12 +1096,22 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args, env: {}) + def run_git(args, chdir: path, env: {}, nice: false, &block) + cmd = [Gitlab.config.git.bin_path, *args] + cmd.unshift("nice") if nice circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path, env) + popen(cmd, chdir, env, &block) end end + def run_git!(args, chdir: path, env: {}, nice: false, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) + + raise GitError, output unless status.zero? + + output + end + # Refactoring aid; allows us to copy code from app/models/repository.rb def run_git_with_timeout(args, timeout, env: {}) circuit_breaker.perform do @@ -1115,36 +1124,27 @@ module Gitlab Gitlab::Git::Commit.find(self, ref) end - # Refactoring aid; allows us to copy code from app/models/repository.rb - def empty_repo? - !exists? || !has_visible_content? - end - - # - # Git repository can contains some hidden refs like: - # /refs/notes/* - # /refs/git-as-svn/* - # /refs/pulls/* - # This refs by default not visible in project page and not cloned to client side. - # - # This method return true if repository contains some content visible in project page. - # - def has_visible_content? - return @has_visible_content if defined?(@has_visible_content) - - @has_visible_content = has_local_branches? + def empty? + !has_visible_content? end - def fetch(remote = 'origin') - args = %W(#{Gitlab.config.git.bin_path} fetch #{remote}) - - popen(args, @path).last.zero? + # Like all public `Gitlab::Git::Repository` methods, this method is part + # of `Repository`'s interface through `method_missing`. + # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and + # takes some extra attributes, so we qualify this method name to prevent confusion. + def fetch_remote_without_shell(remote = 'origin') + run_git(['fetch', remote]).last.zero? end def blob_at(sha, path) Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) end + # Items should be of format [[commit_id, path], [commit_id1, path1]] + def batch_blobs(items, blob_size_limit: nil) + Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) + end + def commit_index(user, branch_name, index, options) committer = user_to_committer(user) @@ -1159,6 +1159,76 @@ module Gitlab end end + def fsck + gitaly_migrate(:git_fsck) do |is_enabled| + msg, status = if is_enabled + gitaly_fsck + else + shell_fsck + end + + raise GitError.new("Could not fsck repository: #{msg}") unless status.zero? + end + end + + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repository.path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + + def rebase_in_progress?(rebase_id) + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + + def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) + squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) + env = git_env_for_user(user).merge( + 'GIT_AUTHOR_NAME' => author.name, + 'GIT_AUTHOR_EMAIL' => author.email + ) + diff_range = "#{start_sha}...#{end_sha}" + diff_files = run_git!( + %W(diff --name-only --diff-filter=a --binary #{diff_range}) + ).chomp + + with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do + # Apply diff of the `diff_range` to the worktree + diff = run_git!(%W(diff --binary #{diff_range})) + run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + stdin.write(diff) + end + + # Commit the `diff_range` diff + run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + + # Return the squash sha. May print a warning for ambiguous refs, but + # we can ignore that with `--quiet` and just take the SHA, if present. + # HEAD here always refers to the current HEAD commit, even if there is + # another ref called HEAD. + run_git!( + %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env + ).chomp + end + end + + def squash_in_progress?(squash_id) + fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1195,6 +1265,94 @@ module Gitlab private + def fresh_worktree?(path) + File.exist?(path) && !clean_stuck_worktree(path) + end + + def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) + base_args = %w(worktree add --detach) + + # Note that we _don't_ want to test for `.present?` here: If the caller + # passes an non nil empty value it means it still wants sparse checkout + # but just isn't interested in any file, perhaps because it wants to + # checkout files in by a changeset but that changeset only adds files. + if sparse_checkout_files + # Create worktree without checking out + run_git!(base_args + ['--no-checkout', worktree_path], env: env) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path) + + configure_sparse_checkout(worktree_git_path, sparse_checkout_files) + + # After sparse checkout configuration, checkout `branch` in worktree + run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) + else + # Create worktree and checkout `branch` in it + run_git!(base_args + [worktree_path, branch], env: env) + end + + yield + ensure + FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) + FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) + end + + def clean_stuck_worktree(path) + return false unless File.mtime(path) < 15.minutes.ago + + FileUtils.rm_rf(path) + true + end + + # Adding a worktree means checking out the repository. For large repos, + # this can be very expensive, so set up sparse checkout for the worktree + # to only check out the files we're interested in. + def configure_sparse_checkout(worktree_git_path, files) + run_git!(%w(config core.sparseCheckout true)) + + return if files.empty? + + worktree_info_path = File.join(worktree_git_path, 'info') + FileUtils.mkdir_p(worktree_info_path) + File.write(File.join(worktree_info_path, 'sparse-checkout'), files) + end + + def gitaly_fsck + gitaly_repository_client.fsck + end + + def shell_fsck + run_git(%W[--git-dir=#{path} fsck], nice: true) + end + + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) + with_repo_branch_commit(source_repository, source_branch) do |commit| + if commit + write_ref(local_ref, commit.sha) + true + else + false + end + end + end + + def worktree_path(prefix, id) + id = id.to_s + raise ArgumentError, "worktree id can't be empty" unless id.present? + raise ArgumentError, "worktree id can't contain slashes " if id.include?("/") + + File.join(path, 'gitlab-worktree', "#{prefix}-#{id}") + end + + def git_env_for_user(user) + { + 'GIT_COMMITTER_NAME' => user.name, + 'GIT_COMMITTER_EMAIL' => user.email, + 'GL_ID' => Gitlab::GlId.gl_id(user), + 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, + 'GL_REPOSITORY' => gl_repository + } + end + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 @@ -1212,11 +1370,25 @@ module Gitlab sort_branches(branches, sort_by) end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 def git_merged_branch_names(branch_names = []) - lines = run_git(['branch', '--merged', root_ref] + branch_names) - .first.lines + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha + + git_arguments = + %W[branch --merged #{root_sha} + --format=%(refname:short)\ %(objectname)] + branch_names - lines.map(&:strip) + lines = run_git(git_arguments).first.lines + + lines.each_with_object([]) do |line, branches| + name, sha = line.strip.split(' ', 2) + + branches << name if sha != root_sha + end end def log_using_shell?(options) @@ -1370,6 +1542,7 @@ module Gitlab end return nil unless tmp_entry.type == :tree + tmp_entry = tmp_entry[dir] end end @@ -1490,6 +1663,7 @@ module Gitlab # Ref names must start with `refs/`. def rugged_ref_exists?(ref_name) raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/') + rugged.references.exist?(ref_name) rescue Rugged::ReferenceError false @@ -1556,6 +1730,7 @@ module Gitlab Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") end @@ -1593,6 +1768,28 @@ module Gitlab end end + def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + revert_tree_id = check_revert_content(commit, start_commit.sha) + raise CreateTreeError unless revert_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: committer, + committer: committer, + tree: revert_tree_id, + parents: [start_commit.sha]) + end + end + def gitaly_add_branch(branch_name, user, target) gitaly_operation_client.user_create_branch(branch_name, user, target) rescue GRPC::FailedPrecondition => ex @@ -1609,28 +1806,54 @@ module Gitlab raise InvalidRef, ex end + def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) + raise CreateTreeError unless cherry_pick_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: { + email: commit.author_email, + name: commit.author_name, + time: commit.authored_date + }, + committer: committer, + tree: cherry_pick_tree_id, + parents: [start_commit.sha]) + end + end + + def check_cherry_pick_content(target_commit, source_sha) + args = [target_commit.sha, source_sha] + args << 1 if target_commit.merge_commit? + + cherry_pick_index = rugged.cherrypick_commit(*args) + return false if cherry_pick_index.conflicts? + + tree_id = cherry_pick_index.write_tree(rugged) + return false unless diff_exists?(source_sha, tree_id) + + tree_id + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) end def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) - gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) - gitaly_address = gitaly_resolver.address(source_repository.storage) - gitaly_token = gitaly_resolver.token(source_repository.storage) - - request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository) - env = { - 'GITALY_ADDRESS' => gitaly_address, - 'GITALY_PAYLOAD' => request.to_json, - 'GITALY_WD' => Dir.pwd, - 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" - } - env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? - args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) - run_git(args, env: env) + run_git(args, env: source_repository.fetch_env) end def gitaly_ff_merge(user, source_sha, target_branch) diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index 637e7a0659c..392bef69e80 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -1,38 +1,47 @@ module Gitlab module Git module RepositoryMirroring - IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze - IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze - MIRROR_REMOTE = 'mirror'.freeze + REFMAPS = { + # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror` + all_refs: '+refs/*:refs/*', + heads: '+refs/heads/*:refs/heads/*', + tags: '+refs/tags/*:refs/tags/*' + }.freeze RemoteError = Class.new(StandardError) - def set_remote_as_mirror(remote_name) - # This is used to define repository as equivalent as "git clone --mirror" - rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*' - rugged.config["remote.#{remote_name}.mirror"] = true - rugged.config["remote.#{remote_name}.prune"] = true - end - - def set_import_remote_as_mirror(remote_name) - # Add first fetch with Rugged so it does not create its own. - rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS - - add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) + def set_remote_as_mirror(remote_name, refmap: :all_refs) + set_remote_refmap(remote_name, refmap) rugged.config["remote.#{remote_name}.mirror"] = true rugged.config["remote.#{remote_name}.prune"] = true end - def add_remote_fetch_config(remote_name, refspec) - run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + def set_remote_refmap(remote_name, refmap) + Array(refmap).each_with_index do |refspec, i| + refspec = REFMAPS[refspec] || refspec + + # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it. + # To make sure we start from scratch, we set the first using rugged, and use `git` for any others + if i == 0 + rugged.config["remote.#{remote_name}.fetch"] = refspec + else + run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + end + end end - def fetch_mirror(url) - add_remote(MIRROR_REMOTE, url) - set_remote_as_mirror(MIRROR_REMOTE) - fetch(MIRROR_REMOTE) - remove_remote(MIRROR_REMOTE) + # Like all_refs public `Gitlab::Git::Repository` methods, this method is part + # of `Repository`'s interface through `method_missing`. + # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and + # takes some extra attributes, so we qualify this method name to prevent confusion. + def fetch_as_mirror_without_shell(url) + remote_name = "tmp-#{SecureRandom.hex}" + add_remote(remote_name, url) + set_remote_as_mirror(remote_name) + fetch_remote_without_shell(remote_name) + ensure + remove_remote(remote_name) if remote_name end def remote_tags(remote) @@ -78,7 +87,7 @@ module Gitlab def list_remote_tags(remote) tag_list, exit_code, error = nil - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote}) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote}) Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| tag_list = stdout.read @@ -88,7 +97,7 @@ module Gitlab raise RemoteError, error unless exit_code.zero? - tag_list.split('\n') + tag_list.split("\n") end end end diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb index 99518c9b1e4..5933312b0b5 100644 --- a/lib/gitlab/git/storage.rb +++ b/lib/gitlab/git/storage.rb @@ -15,6 +15,7 @@ module Gitlab Failing = Class.new(Inaccessible) REDIS_KEY_PREFIX = 'storage_accessible:'.freeze + REDIS_KNOWN_KEYS = "#{REDIS_KEY_PREFIX}known_keys_set".freeze def self.redis Gitlab::Redis::SharedState diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb new file mode 100644 index 00000000000..de63cb4b40c --- /dev/null +++ b/lib/gitlab/git/storage/checker.rb @@ -0,0 +1,98 @@ +module Gitlab + module Git + module Storage + class Checker + include CircuitBreakerSettings + + attr_reader :storage_path, :storage, :hostname, :logger + + def self.check_all(logger = Rails.logger) + threads = Gitlab.config.repositories.storages.keys.map do |storage_name| + Thread.new do + Thread.current[:result] = new(storage_name, logger).check_with_lease + end + end + + threads.map do |thread| + thread.join + thread[:result] + end + end + + def initialize(storage, logger = Rails.logger) + @storage = storage + config = Gitlab.config.repositories.storages[@storage] + @storage_path = config['path'] + @logger = logger + + @hostname = Gitlab::Environment.hostname + end + + def check_with_lease + lease_key = "storage_check:#{cache_key}" + lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout) + result = { storage: storage, success: nil } + + if uuid = lease.try_obtain + result[:success] = check + + Gitlab::ExclusiveLease.cancel(lease_key, uuid) + else + logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running") + end + + result + end + + def check + if Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries) + track_storage_accessible + true + else + track_storage_inaccessible + logger.error("#{hostname}: #{storage}: Not accessible.") + false + end + end + + private + + def track_storage_inaccessible + first_failure = current_failure_info.first_failure || Time.now + last_failure = Time.now + + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :first_failure, first_failure.to_i) + redis.hset(cache_key, :last_failure, last_failure.to_i) + redis.hincrby(cache_key, :failure_count, 1) + redis.expire(cache_key, failure_reset_time) + maintain_known_keys(redis) + end + end + end + + def track_storage_accessible + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :first_failure, nil) + redis.hset(cache_key, :last_failure, nil) + redis.hset(cache_key, :failure_count, 0) + maintain_known_keys(redis) + end + end + end + + def maintain_known_keys(redis) + expire_time = Time.now.to_i + failure_reset_time + redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) + redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) + end + + def current_failure_info + FailureInfo.load(cache_key) + end + end + end + end +end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index be7598ef011..898bb1b65be 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -4,24 +4,11 @@ module Gitlab class CircuitBreaker include CircuitBreakerSettings - FailureInfo = Struct.new(:last_failure, :failure_count) - attr_reader :storage, - :hostname, - :storage_path - - delegate :last_failure, :failure_count, to: :failure_info - - def self.reset_all! - pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*" + :hostname - Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.scan_each(match: pattern).to_a - redis.del(*all_storage_keys) unless all_storage_keys.empty? - end - - RequestStore.delete(:circuitbreaker_cache) - end + delegate :last_failure, :failure_count, :no_failures?, + to: :failure_info def self.for_storage(storage) cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do @@ -48,9 +35,6 @@ module Gitlab def initialize(storage, hostname) @storage = storage @hostname = hostname - - config = Gitlab.config.repositories.storages[@storage] - @storage_path = config['path'] end def perform @@ -67,15 +51,6 @@ module Gitlab failure_count > failure_count_threshold end - def backing_off? - return false if no_failures? - - recent_failure = last_failure > failure_wait_time.seconds.ago - too_many_failures = failure_count > backoff_threshold - - recent_failure && too_many_failures - end - private # The circuitbreaker can be enabled for the entire fleet using a Feature @@ -88,82 +63,13 @@ module Gitlab end def failure_info - @failure_info ||= get_failure_info - end - - # Memoizing the `storage_available` call means we only do it once per - # request when the storage is available. - # - # When the storage appears not available, and the memoized value is `false` - # we might want to try again. - def storage_available? - return @storage_available if @storage_available - - if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck - .storage_available?(storage_path, storage_timeout, access_retries) - track_storage_accessible - else - track_storage_inaccessible - end - - @storage_available + @failure_info ||= FailureInfo.load(cache_key) end def check_storage_accessible! if circuit_broken? raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) end - - if backing_off? - raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time) - end - - unless storage_available? - raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time) - end - end - - def no_failures? - last_failure.blank? && failure_count == 0 - end - - def track_storage_inaccessible - @failure_info = FailureInfo.new(Time.now, failure_count + 1) - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :last_failure, last_failure.to_i) - redis.hincrby(cache_key, :failure_count, 1) - redis.expire(cache_key, failure_reset_time) - end - end - end - - def track_storage_accessible - return if no_failures? - - @failure_info = FailureInfo.new(nil, 0) - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :last_failure, nil) - redis.hset(cache_key, :failure_count, 0) - end - end - end - - def get_failure_info - last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| - redis.hmget(cache_key, :last_failure, :failure_count) - end - - last_failure = Time.at(last_failure.to_i) if last_failure.present? - - FailureInfo.new(last_failure, failure_count.to_i) - end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end end end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb index 257fe8cd8f0..c9e225f187d 100644 --- a/lib/gitlab/git/storage/circuit_breaker_settings.rb +++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb @@ -6,10 +6,6 @@ module Gitlab application_settings.circuitbreaker_failure_count_threshold end - def failure_wait_time - application_settings.circuitbreaker_failure_wait_time - end - def failure_reset_time application_settings.circuitbreaker_failure_reset_time end @@ -22,8 +18,12 @@ module Gitlab application_settings.circuitbreaker_access_retries end - def backoff_threshold - application_settings.circuitbreaker_backoff_threshold + def check_interval + application_settings.circuitbreaker_check_interval + end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end private diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb new file mode 100644 index 00000000000..387279c110d --- /dev/null +++ b/lib/gitlab/git/storage/failure_info.rb @@ -0,0 +1,39 @@ +module Gitlab + module Git + module Storage + class FailureInfo + attr_accessor :first_failure, :last_failure, :failure_count + + def self.reset_all! + Gitlab::Git::Storage.redis.with do |redis| + all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) + redis.del(*all_storage_keys) unless all_storage_keys.empty? + end + + RequestStore.delete(:circuitbreaker_cache) + end + + def self.load(cache_key) + first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| + redis.hmget(cache_key, :first_failure, :last_failure, :failure_count) + end + + last_failure = Time.at(last_failure.to_i) if last_failure.present? + first_failure = Time.at(first_failure.to_i) if first_failure.present? + + new(first_failure, last_failure, failure_count.to_i) + end + + def initialize(first_failure, last_failure, failure_count) + @first_failure = first_failure + @last_failure = last_failure + @failure_count = failure_count + end + + def no_failures? + first_failure.blank? && last_failure.blank? && failure_count == 0 + end + end + end + end +end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 7049772fe3b..90bbe85fd37 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -4,8 +4,8 @@ module Gitlab class Health attr_reader :storage_name, :info - def self.pattern_for_storage(storage_name) - "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:*" + def self.prefix_for_storage(storage_name) + "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:" end def self.for_all_storages @@ -25,26 +25,15 @@ module Gitlab private_class_method def self.all_keys_for_storages(storage_names, redis) keys_per_storage = {} + all_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - redis.pipelined do - storage_names.each do |storage_name| - pattern = pattern_for_storage(storage_name) - matched_keys = redis.scan_each(match: pattern) + storage_names.each do |storage_name| + prefix = prefix_for_storage(storage_name) - keys_per_storage[storage_name] = matched_keys - end + keys_per_storage[storage_name] = all_keys.select { |key| key.starts_with?(prefix) } end - # We need to make sure each lazy-loaded `Enumerator` for matched keys - # is loaded into an array. - # - # Otherwise it would be loaded in the second `Redis#pipelined` block - # within `.load_for_keys`. In this pipelined call, the active - # Redis-client changes again, so the values would not be available - # until the end of that pipelined-block. - keys_per_storage.each do |storage_name, key_future| - keys_per_storage[storage_name] = key_future.to_a - end + keys_per_storage end private_class_method def self.load_for_keys(keys_per_storage, redis) diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb index a12d52d295f..261c936c689 100644 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -11,6 +11,9 @@ module Gitlab # These will always have nil values attr_reader :storage_path + delegate :last_failure, :failure_count, :no_failures?, + to: :failure_info + def initialize(storage, hostname, error: nil) @storage = storage @hostname = hostname @@ -29,16 +32,17 @@ module Gitlab false end - def last_failure - circuit_broken? ? Time.now : nil - end - - def failure_count - circuit_broken? ? failure_count_threshold : 0 - end - def failure_info - Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count) + @failure_info ||= + if circuit_broken? + Gitlab::Git::Storage::FailureInfo.new(Time.now, + Time.now, + failure_count_threshold) + else + Gitlab::Git::Storage::FailureInfo.new(nil, + nil, + 0) + end end end end diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index e6b61417de1..e573cd0e143 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -8,7 +8,12 @@ module Gitlab end def self.from_gitaly(gitaly_user) - new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new( + gitaly_user.gl_username, + Gitlab::EncodingHelper.encode!(gitaly_user.name), + Gitlab::EncodingHelper.encode!(gitaly_user.email), + gitaly_user.gl_id + ) end def initialize(username, name, email, gl_id) @@ -23,7 +28,7 @@ module Gitlab end def to_gitaly - Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email) + Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b) end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index fe901d049d4..d4a53d32c28 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -48,15 +48,24 @@ module Gitlab end def update_page(page_path, title, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) - nil + @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| + if is_enabled + gitaly_update_page(page_path, title, format, content, commit_details) + gollum_wiki.clear_cache + else + gollum_update_page(page_path, title, format, content, commit_details) + end + end end - def pages - gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + def pages(limit: nil) + @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + if is_enabled + gitaly_get_all_pages + else + gollum_get_all_pages(limit: limit) + end + end end def page(title:, version: nil, dir: nil) @@ -79,14 +88,23 @@ module Gitlab end end - def page_versions(page_path) + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def page_versions(page_path, options = {}) current_page = gollum_page_by_path(page_path) - current_page.versions.map do |gollum_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) - new_version(gollum_page, gollum_git_commit.id) + + commits_from_page(current_page, options).map do |gitlab_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) + Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) end end + def count_page_versions(page_path) + @repository.count_commits(ref: 'HEAD', path: page_path) + end + def preview_slug(title, format) # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid # using Rugged through a Gollum::Wiki instance @@ -101,6 +119,22 @@ module Gitlab private + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def commits_from_page(gollum_page, options = {}) + unless options[:limit] + options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page + options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i + end + + @repository.log(ref: gollum_page.last_version.id, + path: gollum_page.path, + limit: options[:limit], + offset: options[:offset]) + end + def gollum_wiki @gollum_wiki ||= Gollum::Wiki.new(@repository.path) end @@ -117,8 +151,17 @@ module Gitlab end def new_version(gollum_page, commit_id) - commit = Gitlab::Git::Commit.find(@repository, commit_id) - Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format) + end + + def version(commit_id) + commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) } + + if RequestStore.active? + RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } + else + commit_find_proc.call + end end def assert_type!(object, klass) @@ -149,6 +192,14 @@ module Gitlab nil end + def gollum_update_page(page_path, title, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + nil + end + def gollum_find_page(title:, version: nil, dir: nil) if version version = Gitlab::Git::Commit.find(@repository, version).id @@ -168,10 +219,18 @@ module Gitlab Gitlab::Git::WikiFile.new(gollum_file) end + def gollum_get_all_pages(limit: nil) + gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) } + end + def gitaly_write_page(name, format, content, commit_details) gitaly_wiki_client.write_page(name, format, content, commit_details) end + def gitaly_update_page(page_path, title, format, content, commit_details) + gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) + end + def gitaly_delete_page(page_path, commit_details) gitaly_wiki_client.delete_page(page_path, commit_details) end @@ -189,6 +248,12 @@ module Gitlab Gitlab::Git::WikiFile.new(wiki_file) end + + def gitaly_get_all_pages + gitaly_wiki_client.get_all_pages.map do |wiki_page, version| + Gitlab::Git::WikiPage.new(wiki_page, version) + end + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 8998c4b1a83..9d7d921bb9c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -166,7 +166,7 @@ module Gitlab end if Gitlab::Database.read_only? - raise UnauthorizedError, ERROR_MESSAGES[:cannot_push_to_read_only] + raise UnauthorizedError, push_to_read_only_message end if deploy_key @@ -280,5 +280,9 @@ module Gitlab UserAccess.new(user, project: project) end end + + def push_to_read_only_message + ERROR_MESSAGES[:cannot_push_to_read_only] + end end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 98f1f45b338..1c9477e84b2 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -19,10 +19,14 @@ module Gitlab end if Gitlab::Database.read_only? - raise UnauthorizedError, ERROR_MESSAGES[:read_only] + raise UnauthorizedError, push_to_read_only_message end true end + + def push_to_read_only_message + ERROR_MESSAGES[:read_only] + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 0b35a787e07..b753ac46291 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -27,18 +27,42 @@ module Gitlab end SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - MAXIMUM_GITALY_CALLS = 30 + MAXIMUM_GITALY_CALLS = 35 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new - private_constant :MUTEX + METRICS_MUTEX = Mutex.new + private_constant :MUTEX, :METRICS_MUTEX class << self - attr_accessor :query_time, :migrate_histogram + attr_accessor :query_time end self.query_time = 0 - self.migrate_histogram = Gitlab::Metrics.histogram(:gitaly_migrate_call_duration, "Gitaly migration call execution timings") + + def self.migrate_histogram + @migrate_histogram ||= + METRICS_MUTEX.synchronize do + # If a thread was blocked on the mutex, the value was set already + return @migrate_histogram if @migrate_histogram + + Gitlab::Metrics.histogram(:gitaly_migrate_call_duration_seconds, + "Gitaly migration call execution timings", + gitaly_enabled: nil, feature: nil) + end + end + + def self.gitaly_call_histogram + @gitaly_call_histogram ||= + METRICS_MUTEX.synchronize do + # If a thread was blocked on the mutex, the value was set already + return @gitaly_call_histogram if @gitaly_call_histogram + + Gitlab::Metrics.histogram(:gitaly_controller_action_duration_seconds, + "Gitaly endpoint histogram by controller and action combination", + Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil)) + end + end def self.stub(name, storage) MUTEX.synchronize do @@ -75,6 +99,10 @@ module Gitlab address end + def self.address_metadata(storage) + Base64.strict_encode64(JSON.dump({ storage => { 'address' => address(storage), 'token' => token(storage) } })) + end + # All Gitaly RPC call sites should use GitalyClient.call. This method # makes sure that per-request authentication headers are set. # @@ -89,18 +117,30 @@ module Gitlab # kwargs.merge(deadline: Time.now + 10) # end # - def self.call(storage, service, rpc, request) - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil) + start = Gitlab::Metrics::System.monotonic_time enforce_gitaly_request_limits(:call) - kwargs = request_kwargs(storage) + kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? + stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend ensure - self.query_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + duration = Gitlab::Metrics::System.monotonic_time - start + + # Keep track, seperately, for the performance bar + self.query_time += duration + gitaly_call_histogram.observe( + current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), + duration) end - def self.request_kwargs(storage) + def self.current_transaction_labels + Gitlab::Metrics::Transaction.current&.labels || {} + end + private_class_method :current_transaction_labels + + def self.request_kwargs(storage, timeout, remote_storage: nil) encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { 'authorization' => "Bearer #{encoded_token}", @@ -110,8 +150,24 @@ module Gitlab feature_stack = Thread.current[:gitaly_feature_stack] feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature + metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage + + result = { metadata: metadata } - { metadata: metadata } + # nil timeout indicates that we should use the default + timeout = default_timeout if timeout.nil? + + return result unless timeout > 0 + + # Do not use `Time.now` for deadline calculation, since it + # will be affected by Timecop in some tests, but grpc's c-core + # uses system time instead of timecop's time, so tests will fail + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will + # circumvent timecop + deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + result[:deadline] = deadline + + result end def self.token(storage) @@ -172,10 +228,10 @@ module Gitlab feature_stack = Thread.current[:gitaly_feature_stack] ||= [] feature_stack.unshift(feature) begin - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + start = Gitlab::Metrics::System.monotonic_time yield is_enabled ensure - total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + total_time = Gitlab::Metrics::System.monotonic_time - start migrate_histogram.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) feature_stack.shift Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? @@ -280,10 +336,36 @@ module Gitlab s.dup.force_encoding(Encoding::ASCII_8BIT) end + def self.binary_stringio(s) + io = StringIO.new(s || '') + io.set_encoding(Encoding::ASCII_8BIT) + io + end + def self.encode_repeated(a) Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) end + # The default timeout on all Gitaly calls + def self.default_timeout + return 0 if Sidekiq.server? + + timeout(:gitaly_timeout_default) + end + + def self.fast_timeout + timeout(:gitaly_timeout_fast) + end + + def self.medium_timeout + timeout(:gitaly_timeout_medium) + end + + def self.timeout(timeout_name) + Gitlab::CurrentSettings.current_application_settings[timeout_name] + end + private_class_method :timeout + # Count a stack. Used for n+1 detection def self.count_stack return unless RequestStore.active? diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb new file mode 100644 index 00000000000..198a1de91c7 --- /dev/null +++ b/lib/gitlab/gitaly_client/attributes_bag.rb @@ -0,0 +1,31 @@ +module Gitlab + module GitalyClient + # This module expects an `ATTRS` const to be defined on the subclass + # See GitalyClient::WikiFile for an example + module AttributesBag + extend ActiveSupport::Concern + + included do + attr_accessor(*const_get(:ATTRS)) + end + + def initialize(params) + params = params.with_indifferent_access + + attributes.each do |attr| + instance_variable_set("@#{attr}", params[attr]) + end + end + + def ==(other) + attributes.all? do |field| + instance_variable_get("@#{field}") == other.instance_variable_get("@#{field}") + end + end + + def attributes + self.class.const_get(:ATTRS) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index da5505cb2fe..7985f5b5457 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -16,7 +16,7 @@ module Gitlab revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request) + response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map { |d| EncodingHelper.encode!(d.dup) } end @@ -29,7 +29,7 @@ module Gitlab child_id: child_id ) - GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value + GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value end def diff(from, to, options = {}) @@ -77,7 +77,7 @@ module Gitlab limit: limit.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) + response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) entry = nil data = '' @@ -102,7 +102,7 @@ module Gitlab path: path.present? ? GitalyClient.encode(path) : '.' ) - response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) + response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) response.flat_map do |message| message.entries.map do |gitaly_tree_entry| @@ -129,7 +129,7 @@ module Gitlab request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? request.path = options[:path] if options[:path].present? - GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count + GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end def last_commit_for_path(revision, path) @@ -139,7 +139,7 @@ module Gitlab path: GitalyClient.encode(path.to_s) ) - gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request).commit + gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit return unless gitaly_commit Gitlab::Git::Commit.new(@repository, gitaly_commit) @@ -152,7 +152,7 @@ module Gitlab to: to ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request) + response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -165,7 +165,7 @@ module Gitlab ) request.order = opts[:order].upcase if opts[:order].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -179,7 +179,7 @@ module Gitlab offset: offset.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request) + response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -197,7 +197,7 @@ module Gitlab path: GitalyClient.encode(path) ) - response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request) + response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce("") { |memo, msg| memo << msg.data } end @@ -207,7 +207,7 @@ module Gitlab revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) response.commit end @@ -217,7 +217,7 @@ module Gitlab repository: @gitaly_repo, revision: GitalyClient.encode(revision) ) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout) response.sum(&:data) end @@ -227,7 +227,7 @@ module Gitlab repository: @gitaly_repo, revision: GitalyClient.encode(revision) ) - GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) + GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end def find_commits(options) @@ -245,11 +245,31 @@ module Gitlab request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request) + response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end + def filter_shas_with_signatures(shas) + request = Gitaly::FilterShasWithSignaturesRequest.new(repository: @gitaly_repo) + + enum = Enumerator.new do |y| + shas.each_slice(20) do |revs| + request.shas = GitalyClient.encode_repeated(revs) + + y.yield request + + request = Gitaly::FilterShasWithSignaturesRequest.new + end + end + + response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum) + + response.flat_map do |msg| + msg.shas.map { |sha| EncodingHelper.encode!(sha) } + end + end + private def call_commit_diff(request_params, options = {}) @@ -259,7 +279,7 @@ module Gitlab request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) GitalyClient::DiffStitcher.new(response) end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index 54df6304865..d98a0ce988f 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -1,21 +1,9 @@ module Gitlab module GitalyClient class Diff - FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze + ATTRS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze - attr_accessor(*FIELDS) - - def initialize(params) - params.each do |key, val| - public_send(:"#{key}=", val) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def ==(other) - FIELDS.all? do |field| - public_send(field) == other.public_send(field) # rubocop:disable GitlabSecurity/PublicSend - end - end + include AttributesBag end end end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb index 65d81dc5d46..da243ee2d1a 100644 --- a/lib/gitlab/gitaly_client/diff_stitcher.rb +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -12,7 +12,7 @@ module Gitlab @rpc_response.each do |diff_msg| if current_diff.nil? - diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) + diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::ATTRS) # gRPC uses frozen strings by default, and we need to have an unfrozen string as it # gets processed further down the line. So we unfreeze the first chunk of the patch # in case it's the only chunk we receive for this diff. diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 526d44a8b77..400a4af363b 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -122,6 +122,64 @@ module Gitlab ).branch_update Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) end + + def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + call_cherry_pick_or_revert(:cherry_pick, + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository) + end + + def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + call_cherry_pick_or_revert(:revert, + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository) + end + + private + + def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize + + request = request_class.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + commit: commit.to_gitaly_commit, + branch_name: GitalyClient.encode(branch_name), + message: GitalyClient.encode(message), + start_branch_name: GitalyClient.encode(start_branch_name.to_s), + start_repository: start_repository.gitaly_repository + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :"user_#{rpc}", + request, + remote_storage: start_repository.storage + ) + + handle_cherry_pick_or_revert_response(response) + end + + def handle_cherry_pick_or_revert_response(response) + if response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + elsif response.commit_error.presence + raise Gitlab::Git::CommitError, response.commit_error + elsif response.create_tree_error.presence + raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error + else + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + end end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index b0c73395cb1..066e4e183c0 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -46,7 +46,8 @@ module Gitlab commit_id: commit_id, prefix: ref_prefix ) - encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup) + response = GitalyClient.call(@storage, :ref_service, :find_ref_name, request, timeout: GitalyClient.medium_timeout) + encode!(response.name.dup) end def count_tag_names @@ -126,6 +127,15 @@ module Gitlab GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) end + def delete_refs(except_with_prefixes:) + request = Gitaly::DeleteRefsRequest.new( + repository: @gitaly_repo, + except_with_prefix: except_with_prefixes + ) + + GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + end + private def consume_refs_response(response) @@ -137,6 +147,7 @@ module Gitlab enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index cef692d3c2a..a477d618f63 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -10,7 +10,9 @@ module Gitlab def exists? request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repository_exists, request).exists + response = GitalyClient.call(@storage, :repository_service, :repository_exists, request, timeout: GitalyClient.fast_timeout) + + response.exists end def garbage_collect(create_bitmap) @@ -30,7 +32,8 @@ module Gitlab def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repository_size, request).size + response = GitalyClient.call(@storage, :repository_service, :repository_size, request) + response.size end def apply_gitattributes(revision) @@ -61,10 +64,40 @@ module Gitlab def has_local_branches? request = Gitaly::HasLocalBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request) + response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request, timeout: GitalyClient.fast_timeout) response.value end + + def fetch_source_branch(source_repository, source_branch, local_ref) + request = Gitaly::FetchSourceBranchRequest.new( + repository: @gitaly_repo, + source_repository: source_repository.gitaly_repository, + source_branch: source_branch.b, + target_ref: local_ref.b + ) + + response = GitalyClient.call( + @storage, + :repository_service, + :fetch_source_branch, + request, + remote_storage: source_repository.storage + ) + + response.result + end + + def fsck + request = Gitaly::FsckRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :fsck, request) + + if response.error.empty? + return "", 0 + else + return response.error.b, 1 + end + end end end end diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb index a2e415864e6..47c60c92484 100644 --- a/lib/gitlab/gitaly_client/wiki_file.rb +++ b/lib/gitlab/gitaly_client/wiki_file.rb @@ -1,17 +1,9 @@ module Gitlab module GitalyClient class WikiFile - FIELDS = %i(name mime_type path raw_data).freeze + ATTRS = %i(name mime_type path raw_data).freeze - attr_accessor(*FIELDS) - - def initialize(params) - params = params.with_indifferent_access - - FIELDS.each do |field| - instance_variable_set("@#{field}", params[field]) - end - end + include AttributesBag end end end diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index 8226278d5f6..7339468e911 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -1,16 +1,16 @@ module Gitlab module GitalyClient class WikiPage - FIELDS = %i(title format url_path path name historical raw_data).freeze + ATTRS = %i(title format url_path path name historical raw_data).freeze - attr_accessor(*FIELDS) + include AttributesBag def initialize(params) - params = params.with_indifferent_access + super - FIELDS.each do |field| - instance_variable_set("@#{field}", params[field]) - end + # All gRPC strings in a response are frozen, so we get an unfrozen + # version here so appending to `raw_data` doesn't blow up. + @raw_data = @raw_data.dup end def historical? diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 15f0f30d303..337d225d081 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -18,12 +18,11 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - strio = StringIO.new(content) + strio = GitalyClient.binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? - chunk = strio.read(MAX_MSG_SIZE) - request.content = GitalyClient.encode(chunk) + request.content = strio.read(MAX_MSG_SIZE) y.yield request @@ -37,6 +36,30 @@ module Gitlab end end + def update_page(page_path, title, format, content, commit_details) + request = Gitaly::WikiUpdatePageRequest.new( + repository: @gitaly_repo, + page_path: GitalyClient.encode(page_path), + title: GitalyClient.encode(title), + format: format.to_s, + commit_details: gitaly_commit_details(commit_details) + ) + + strio = GitalyClient.binary_stringio(content) + + enum = Enumerator.new do |y| + until strio.eof? + request.content = strio.read(MAX_MSG_SIZE) + + y.yield request + + request = Gitaly::WikiUpdatePageRequest.new + end + end + + GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum) + end + def delete_page(page_path, commit_details) request = Gitaly::WikiDeletePageRequest.new( repository: @gitaly_repo, @@ -56,28 +79,24 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) - wiki_page = version = nil - response.each do |message| - page = message.page - next unless page + wiki_page_from_iterator(response) + end - if wiki_page - wiki_page.raw_data << page.raw_data - else - wiki_page = GitalyClient::WikiPage.new(page.to_h) - # All gRPC strings in a response are frozen, so we get - # an unfrozen version here so appending in the else clause below doesn't blow up. - wiki_page.raw_data = wiki_page.raw_data.dup + def get_all_pages + request = Gitaly::WikiGetAllPagesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request) + pages = [] - version = Gitlab::Git::WikiPageVersion.new( - Gitlab::Git::Commit.decorate(@repository, page.version.commit), - page.version.format - ) - end + loop do + page, version = wiki_page_from_iterator(response) { |message| message.end_of_page } + + break unless page && version + + pages << [page, version] end - [wiki_page, version] + pages end def find_file(name, revision) @@ -108,6 +127,35 @@ module Gitlab private + # If a block is given and the yielded value is true, iteration will be + # stopped early at that point; else the iterator is consumed entirely. + # The iterator is traversed with `next` to allow resuming the iteration. + def wiki_page_from_iterator(iterator) + wiki_page = version = nil + + while message = iterator.next + break if block_given? && yield(message) + + page = message.page + next unless page + + if wiki_page + wiki_page.raw_data << page.raw_data + else + wiki_page = GitalyClient::WikiPage.new(page.to_h) + + version = Gitlab::Git::WikiPageVersion.new( + Gitlab::Git::Commit.decorate(@repository, page.version.commit), + page.version.format + ) + end + end + + [wiki_page, version] + rescue StopIteration + [wiki_page, version] + end + def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( name: GitalyClient.encode(commit_details.name), diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb new file mode 100644 index 00000000000..65b5e30c70f --- /dev/null +++ b/lib/gitlab/github_import.rb @@ -0,0 +1,38 @@ +module Gitlab + module GithubImport + def self.refmap + [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head'] + end + + def self.new_client_for(project, token: nil, parallel: true) + token_to_use = token || project.import_data&.credentials&.fetch(:user) + + Client.new(token_to_use, parallel: parallel) + end + + # Inserts a raw row and returns the ID of the inserted row. + # + # attributes - The attributes/columns to set. + # relation - An ActiveRecord::Relation to use for finding the ID of the row + # when using MySQL. + def self.insert_and_return_id(attributes, relation) + # We use bulk_insert here so we can bypass any queries executed by + # callbacks or validation rules, as doing this wouldn't scale when + # importing very large projects. + result = Gitlab::Database + .bulk_insert(relation.table_name, [attributes], return_ids: true) + + # MySQL doesn't support returning the IDs of a bulk insert in a way that + # is not a pain, so in this case we'll issue an extra query instead. + result.first || + relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + end + + # Returns the ID of the ghost user. + def self.ghost_user_id + key = 'github-import/ghost-user-id' + + Caching.read_integer(key) || Caching.write(key, User.select(:id).ghost.id) + end + end +end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb new file mode 100644 index 00000000000..147597289cf --- /dev/null +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module BulkImporting + # Builds and returns an Array of objects to bulk insert into the + # database. + # + # enum - An Enumerable that returns the objects to turn into database + # rows. + def build_database_rows(enum) + enum.each_with_object([]) do |(object, _), rows| + rows << build(object) unless already_imported?(object) + end + end + + # Bulk inserts the given rows into the database. + def bulk_insert(model, rows, batch_size: 100) + rows.each_slice(batch_size) do |slice| + Gitlab::Database.bulk_insert(model.table_name, slice) + end + end + end + end +end diff --git a/lib/gitlab/github_import/caching.rb b/lib/gitlab/github_import/caching.rb new file mode 100644 index 00000000000..b08f133794f --- /dev/null +++ b/lib/gitlab/github_import/caching.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Caching + # The default timeout of the cache keys. + TIMEOUT = 24.hours.to_i + + WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze + local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2] + local existing = tonumber(redis.call("get", key)) + + if existing == nil or value > existing then + redis.call("set", key, value) + redis.call("expire", key, ttl) + return true + else + return false + end + EOF + + # Reads a cache key. + # + # If the key exists and has a non-empty value its TTL is refreshed + # automatically. + # + # raw_key - The cache key to read. + # timeout - The new timeout of the key if the key is to be refreshed. + def self.read(raw_key, timeout: TIMEOUT) + key = cache_key_for(raw_key) + value = Redis::Cache.with { |redis| redis.get(key) } + + if value.present? + # We refresh the expiration time so frequently used keys stick + # around, removing the need for querying the database as much as + # possible. + # + # A key may be empty when we looked up a GitHub user (for example) but + # did not find a matching GitLab user. In that case we _don't_ want to + # refresh the TTL so we automatically pick up the right data when said + # user were to register themselves on the GitLab instance. + Redis::Cache.with { |redis| redis.expire(key, timeout) } + end + + value + end + + # Reads an integer from the cache, or returns nil if no value was found. + # + # See Caching.read for more information. + def self.read_integer(raw_key, timeout: TIMEOUT) + value = read(raw_key, timeout: timeout) + + value.to_i if value.present? + end + + # Sets a cache key to the given value. + # + # key - The cache key to write. + # value - The value to set. + # timeout - The time after which the cache key should expire. + def self.write(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.set(key, value, ex: timeout) + end + + value + end + + # Adds a value to a set. + # + # raw_key - The key of the set to add the value to. + # value - The value to add to the set. + # timeout - The new timeout of the key. + def self.set_add(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.multi do |m| + m.sadd(key, value) + m.expire(key, timeout) + end + end + end + + # Returns true if the given value is present in the set. + # + # raw_key - The key of the set to check. + # value - The value to check for. + def self.set_includes?(raw_key, value) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.sismember(key, value) + end + end + + # Sets multiple keys to a given value. + # + # mapping - A Hash mapping the cache keys to their values. + # timeout - The time after which the cache key should expire. + def self.write_multiple(mapping, timeout: TIMEOUT) + Redis::Cache.with do |redis| + redis.multi do |multi| + mapping.each do |raw_key, value| + multi.set(cache_key_for(raw_key), value, ex: timeout) + end + end + end + end + + # Sets the expiration time of a key. + # + # raw_key - The key for which to change the timeout. + # timeout - The new timeout. + def self.expire(raw_key, timeout) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.expire(key, timeout) + end + end + + # Sets a key to the given integer but only if the existing value is + # smaller than the given value. + # + # This method uses a Lua script to ensure the read and write are atomic. + # + # raw_key - The key to set. + # value - The new value for the key. + # timeout - The key timeout in seconds. + # + # Returns true when the key was overwritten, false otherwise. + def self.write_if_greater(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + val = Redis::Cache.with do |redis| + redis + .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout]) + end + + val ? true : false + end + + def self.cache_key_for(raw_key) + "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 0550f9695bd..5da9befa08e 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -1,147 +1,216 @@ +# frozen_string_literal: true + module Gitlab module GithubImport + # HTTP client for interacting with the GitHub API. + # + # This class is basically a fancy wrapped around Octokit while adding some + # functionality to deal with rate limiting and parallel imports. Usage is + # mostly the same as Octokit, for example: + # + # client = GithubImport::Client.new('hunter2') + # + # client.labels.each do |label| + # puts label.name + # end class Client - GITHUB_SAFE_REMAINING_REQUESTS = 100 - GITHUB_SAFE_SLEEP_TIME = 500 + attr_reader :octokit + + # A single page of data and the corresponding page number. + Page = Struct.new(:objects, :number) + + # The minimum number of requests we want to keep available. + # + # We don't use a value of 0 as multiple threads may be using the same + # token in parallel. This could result in all of them hitting the GitHub + # rate limit at once. The threshold is put in place to not hit the limit + # in most cases. + RATE_LIMIT_THRESHOLD = 50 + + # token - The GitHub API token to use. + # + # per_page - The number of objects that should be displayed per page. + # + # parallel - When set to true hitting the rate limit will result in a + # dedicated error being raised. When set to `false` we will + # instead just `sleep()` until the rate limit is reset. Setting + # this value to `true` for parallel importing is crucial as + # otherwise hitting the rate limit will result in a thread + # being blocked in a `sleep()` call for up to an hour. + def initialize(token, per_page: 100, parallel: true) + @octokit = Octokit::Client.new( + access_token: token, + per_page: per_page, + api_endpoint: api_endpoint + ) - attr_reader :access_token, :host, :api_version + @octokit.connection_options[:ssl] = { verify: verify_ssl } - def initialize(access_token, host: nil, api_version: 'v3') - @access_token = access_token - @host = host.to_s.sub(%r{/+\z}, '') - @api_version = api_version - @users = {} + @parallel = parallel + end - if access_token - ::Octokit.auto_paginate = false - end + def parallel? + @parallel end - def api - @api ||= ::Octokit::Client.new( - access_token: access_token, - api_endpoint: api_endpoint, - # If there is no config, we're connecting to github.com and we - # should verify ssl. - connection_options: { - ssl: { verify: config ? config['verify_ssl'] : true } - } - ) + # Returns the details of a GitHub user. + # + # username - The username of the user. + def user(username) + with_rate_limit { octokit.user(username) } end - def client - unless config - raise Projects::ImportService::Error, - 'OAuth configuration for GitHub missing.' - end + # Returns the details of a GitHub repository. + # + # name - The path (in the form `owner/repository`) of the repository. + def repository(name) + with_rate_limit { octokit.repo(name) } + end - @client ||= ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + def labels(*args) + each_object(:labels, *args) end - def authorize_url(redirect_uri) - client.auth_code.authorize_url({ - redirect_uri: redirect_uri, - scope: "repo, user, user:email" - }) + def milestones(*args) + each_object(:milestones, *args) end - def get_token(code) - client.auth_code.get_token(code).token + def releases(*args) + each_object(:releases, *args) end - def method_missing(method, *args, &block) - if api.respond_to?(method) - request(method, *args, &block) - else - super(method, *args, &block) + # Fetches data from the GitHub API and yields a Page object for every page + # of data, without loading all of them into memory. + # + # method - The Octokit method to use for getting the data. + # args - Arguments to pass to the Octokit method. + # + # rubocop: disable GitlabSecurity/PublicSend + def each_page(method, *args, &block) + return to_enum(__method__, method, *args) unless block_given? + + page = + if args.last.is_a?(Hash) && args.last[:page] + args.last[:page] + else + 1 + end + + collection = with_rate_limit { octokit.public_send(method, *args) } + next_url = octokit.last_response.rels[:next] + + yield Page.new(collection, page) + + while next_url + response = with_rate_limit { next_url.get } + next_url = response.rels[:next] + + yield Page.new(response.data, page += 1) end end - def respond_to?(method) - api.respond_to?(method) || super + # Iterates over all of the objects for the given method (e.g. `:labels`). + # + # method - The method to send to Octokit for querying data. + # args - Any arguments to pass to the Octokit method. + def each_object(method, *args, &block) + return to_enum(__method__, method, *args) unless block_given? + + each_page(method, *args) do |page| + page.objects.each do |object| + yield object + end + end end - def user(login) - return nil unless login.present? - return @users[login] if @users.key?(login) + # Yields the supplied block, responding to any rate limit errors. + # + # The exact strategy used for handling rate limiting errors depends on + # whether we are running in parallel mode or not. For more information see + # `#rate_or_wait_for_rate_limit`. + def with_rate_limit + return yield unless rate_limiting_enabled? - @users[login] = api.user(login) - end + request_count_counter.increment - private + raise_or_wait_for_rate_limit unless requests_remaining? - def api_endpoint - if host.present? && api_version.present? - "#{host}/api/#{api_version}" - else - github_options[:site] + begin + yield + rescue Octokit::TooManyRequests + raise_or_wait_for_rate_limit + + # This retry will only happen when running in sequential mode as we'll + # raise an error in parallel mode. + retry end end - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } + # Returns `true` if we're still allowed to perform API calls. + def requests_remaining? + remaining_requests > RATE_LIMIT_THRESHOLD + end + + def remaining_requests + octokit.rate_limit.remaining end - def github_options - if config - config["args"]["client_options"].deep_symbolize_keys + def raise_or_wait_for_rate_limit + rate_limit_counter.increment + + if parallel? + raise RateLimitError else - OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + sleep(rate_limit_resets_in) end end - def rate_limit - api.rate_limit! - # GitHub Rate Limit API returns 404 when the rate limit is - # disabled. In this case we just want to return gracefully - # instead of spitting out an error. - rescue Octokit::NotFound - nil + def rate_limit_resets_in + # We add a few seconds to the rate limit so we don't _immediately_ + # resume when the rate limit resets as this may result in us performing + # a request before GitHub has a chance to reset the limit. + octokit.rate_limit.resets_in + 5 end - def has_rate_limit? - return @has_rate_limit if defined?(@has_rate_limit) - - @has_rate_limit = rate_limit.present? + def rate_limiting_enabled? + @rate_limiting_enabled ||= api_endpoint.include?('.github.com') end - def rate_limit_exceed? - has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + def api_endpoint + custom_api_endpoint || default_api_endpoint end - def rate_limit_sleep_time - rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + def custom_api_endpoint + github_omniauth_provider.dig('args', 'client_options', 'site') end - def request(method, *args, &block) - sleep rate_limit_sleep_time if rate_limit_exceed? + def default_api_endpoint + OmniAuth::Strategies::GitHub.default_options[:client_options][:site] + end - data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend - return data unless data.is_a?(Array) + def verify_ssl + github_omniauth_provider.fetch('verify_ssl', true) + end - last_response = api.last_response + def github_omniauth_provider + @github_omniauth_provider ||= + Gitlab.config.omniauth.providers + .find { |provider| provider.name == 'github' } + .to_h + end - if block_given? - yield data - # api.last_response could change while we're yielding (e.g. fetching labels for each PR) - # so we cache our own last response - each_response_page(last_response, &block) - else - each_response_page(last_response) { |page| data.concat(page) } - data - end + def rate_limit_counter + @rate_limit_counter ||= Gitlab::Metrics.counter( + :github_importer_rate_limit_hits, + 'The number of times we hit the GitHub rate limit when importing projects' + ) end - def each_response_page(last_response) - while last_response.rels[:next] - sleep rate_limit_sleep_time if rate_limit_exceed? - last_response = last_response.rels[:next].get - yield last_response.data if last_response.data.is_a?(Array) - end + def request_count_counter + @request_counter ||= Gitlab::Metrics.counter( + :github_importer_request_count, + 'The number of GitHub API calls performed when importing projects' + ) end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb new file mode 100644 index 00000000000..8274f37d358 --- /dev/null +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class DiffNoteImporter + attr_reader :note, :project, :client, :user_finder + + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note, project, client) + @note = note + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + return unless (mr_id = find_merge_request_id) + + author_id, author_found = user_finder.author_id_for(note) + + note_body = + MarkdownText.format(note.note, note.author, author_found) + + attributes = { + noteable_type: 'MergeRequest', + noteable_id: mr_id, + project_id: project.id, + author_id: author_id, + note: note_body, + system: false, + commit_id: note.commit_id, + line_code: note.line_code, + type: 'LegacyDiffNote', + created_at: note.created_at, + updated_at: note.updated_at, + st_diff: note.diff_hash.to_yaml + } + + # It's possible that during an import we'll insert tens of thousands + # of diff notes. If we were to use the Note/LegacyDiffNote model here + # we'd also have to run additional queries for both validations and + # callbacks, putting a lot of pressure on the database. + # + # To work around this we're using bulk_insert with a single row. This + # allows us to efficiently insert data (even if it's just 1 row) + # without having to use all sorts of hacks to disable callbacks. + Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note. + end + + # Returns the ID of the merge request this note belongs to. + def find_merge_request_id + GithubImport::IssuableFinder.new(project, note).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb new file mode 100644 index 00000000000..966f12c5c2f --- /dev/null +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class DiffNotesImporter + include ParallelScheduling + + def representation_class + Representation::DiffNote + end + + def importer_class + DiffNoteImporter + end + + def sidekiq_worker_class + ImportDiffNoteWorker + end + + def collection_method + :pull_requests_comments + end + + def id_for_already_imported_cache(note) + note.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb b/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb new file mode 100644 index 00000000000..bad064b76c8 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueAndLabelLinksImporter + attr_reader :issue, :project, :client + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + end + + def execute + IssueImporter.import_if_issue(issue, project, client) + LabelLinksImporter.new(issue, project, client).execute + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb new file mode 100644 index 00000000000..31fefebf787 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueImporter + attr_reader :project, :issue, :client, :user_finder, :milestone_finder, + :issuable_finder + + # Imports an issue if it's a regular issue and not a pull request. + def self.import_if_issue(issue, project, client) + new(issue, project, client).execute unless issue.pull_request? + end + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + @milestone_finder = MilestoneFinder.new(project) + @issuable_finder = GithubImport::IssuableFinder.new(project, issue) + end + + def execute + Issue.transaction do + if (issue_id = create_issue) + create_assignees(issue_id) + issuable_finder.cache_database_id(issue_id) + end + end + end + + # Creates a new GitLab issue for the current GitHub issue. + # + # Returns the ID of the created issue as an Integer. If the issue + # couldn't be created this method will return `nil` instead. + def create_issue + author_id, author_found = user_finder.author_id_for(issue) + + description = + MarkdownText.format(issue.description, issue.author, author_found) + + attributes = { + iid: issue.iid, + title: issue.truncated_title, + author_id: author_id, + project_id: project.id, + description: description, + milestone_id: milestone_finder.id_for(issue), + state: issue.state, + created_at: issue.created_at, + updated_at: issue.updated_at + } + + GithubImport.insert_and_return_id(attributes, project.issues) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the issue. + end + + # Stores all issue assignees in the database. + # + # issue_id - The ID of the created issue. + def create_assignees(issue_id) + assignees = [] + + issue.assignees.each do |assignee| + if (user_id = user_finder.user_id_for(assignee)) + assignees << { issue_id: issue_id, user_id: user_id } + end + end + + Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb new file mode 100644 index 00000000000..ac6d0666b3a --- /dev/null +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssuesImporter + include ParallelScheduling + + def importer_class + IssueAndLabelLinksImporter + end + + def representation_class + Representation::Issue + end + + def sidekiq_worker_class + ImportIssueWorker + end + + def collection_method + :issues + end + + def id_for_already_imported_cache(issue) + issue.number + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb new file mode 100644 index 00000000000..2001b7e3482 --- /dev/null +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LabelLinksImporter + attr_reader :issue, :project, :client, :label_finder + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + @label_finder = LabelFinder.new(project) + end + + def execute + create_labels + end + + def create_labels + time = Time.zone.now + rows = [] + target_id = find_target_id + + issue.label_names.each do |label_name| + # Although unlikely it's technically possible for an issue to be + # given a label that was created and assigned after we imported all + # the project's labels. + next unless (label_id = label_finder.id_for(label_name)) + + rows << { + label_id: label_id, + target_id: target_id, + target_type: issue.issuable_type, + created_at: time, + updated_at: time + } + end + + Gitlab::Database.bulk_insert(LabelLink.table_name, rows) + end + + def find_target_id + GithubImport::IssuableFinder.new(project, issue).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb new file mode 100644 index 00000000000..a73033d35ba --- /dev/null +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LabelsImporter + include BulkImporting + + attr_reader :project, :client, :existing_labels + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(project, client) + @project = project + @client = client + @existing_labels = project.labels.pluck(:title).to_set + end + + def execute + bulk_insert(Label, build_labels) + build_labels_cache + end + + def build_labels + build_database_rows(each_label) + end + + def already_imported?(label) + existing_labels.include?(label.name) + end + + def build_labels_cache + LabelFinder.new(project).build_cache + end + + def build(label) + time = Time.zone.now + + { + title: label.name, + color: '#' + label.color, + project_id: project.id, + type: 'ProjectLabel', + created_at: time, + updated_at: time + } + end + + def each_label + client.labels(project.import_source) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb new file mode 100644 index 00000000000..c53480e828a --- /dev/null +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class MilestonesImporter + include BulkImporting + + attr_reader :project, :client, :existing_milestones + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + @existing_milestones = project.milestones.pluck(:iid).to_set + end + + def execute + bulk_insert(Milestone, build_milestones) + build_milestones_cache + end + + def build_milestones + build_database_rows(each_milestone) + end + + def already_imported?(milestone) + existing_milestones.include?(milestone.number) + end + + def build_milestones_cache + MilestoneFinder.new(project).build_cache + end + + def build(milestone) + { + iid: milestone.number, + title: milestone.title, + description: milestone.description, + project_id: project.id, + state: state_for(milestone), + created_at: milestone.created_at, + updated_at: milestone.updated_at + } + end + + def state_for(milestone) + milestone.state == 'open' ? :active : :closed + end + + def each_milestone + client.milestones(project.import_source, state: 'all') + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb new file mode 100644 index 00000000000..c890f2df360 --- /dev/null +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NoteImporter + attr_reader :note, :project, :client, :user_finder + + # note - An instance of `Gitlab::GithubImport::Representation::Note`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note, project, client) + @note = note + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + return unless (noteable_id = find_noteable_id) + + author_id, author_found = user_finder.author_id_for(note) + + note_body = + MarkdownText.format(note.note, note.author, author_found) + + attributes = { + noteable_type: note.noteable_type, + noteable_id: noteable_id, + project_id: project.id, + author_id: author_id, + note: note_body, + system: false, + created_at: note.created_at, + updated_at: note.updated_at + } + + # We're using bulk_insert here so we can bypass any validations and + # callbacks. Running these would result in a lot of unnecessary SQL + # queries being executed when importing large projects. + Gitlab::Database.bulk_insert(Note.table_name, [attributes]) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note. + end + + # Returns the ID of the issue or merge request to create the note for. + def find_noteable_id + GithubImport::IssuableFinder.new(project, note).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb new file mode 100644 index 00000000000..5aec760ea5f --- /dev/null +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NotesImporter + include ParallelScheduling + + def importer_class + NoteImporter + end + + def representation_class + Representation::Note + end + + def sidekiq_worker_class + ImportNoteWorker + end + + def collection_method + :issues_comments + end + + def id_for_already_imported_cache(note) + note.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb new file mode 100644 index 00000000000..49d859f9624 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestImporter + attr_reader :pull_request, :project, :client, :user_finder, + :milestone_finder, :issuable_finder + + # pull_request - An instance of + # `Gitlab::GithubImport::Representation::PullRequest`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(pull_request, project, client) + @pull_request = pull_request + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + @milestone_finder = MilestoneFinder.new(project) + @issuable_finder = + GithubImport::IssuableFinder.new(project, pull_request) + end + + def execute + if (mr_id = create_merge_request) + issuable_finder.cache_database_id(mr_id) + end + end + + # Creates the merge request and returns its ID. + # + # This method will return `nil` if the merge request could not be + # created. + def create_merge_request + author_id, author_found = user_finder.author_id_for(pull_request) + + description = MarkdownText + .format(pull_request.description, pull_request.author, author_found) + + # This work must be wrapped in a transaction as otherwise we can leave + # behind incomplete data in the event of an error. This can then lead + # to duplicate key errors when jobs are retried. + MergeRequest.transaction do + attributes = { + iid: pull_request.iid, + title: pull_request.truncated_title, + description: description, + source_project_id: project.id, + target_project_id: project.id, + source_branch: pull_request.formatted_source_branch, + target_branch: pull_request.target_branch, + state: pull_request.state, + milestone_id: milestone_finder.id_for(pull_request), + author_id: author_id, + assignee_id: user_finder.assignee_id_for(pull_request), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + # When creating merge requests there are a lot of hooks that may + # run, for many different reasons. Many of these hooks (e.g. the + # ones used for rendering Markdown) are completely unnecessary and + # may even lead to transaction timeouts. + # + # To ensure importing pull requests has a minimal impact and can + # complete in a reasonable time we bypass all the hooks by inserting + # the row and then retrieving it. We then only perform the + # additional work that is strictly necessary. + merge_request_id = GithubImport + .insert_and_return_id(attributes, project.merge_requests) + + merge_request = project.merge_requests.find(merge_request_id) + + # These fields are set so we can create the correct merge request + # diffs. + merge_request.source_branch_sha = pull_request.source_branch_sha + merge_request.target_branch_sha = pull_request.target_branch_sha + + merge_request.keep_around_commit + merge_request.merge_request_diffs.create + + merge_request.id + end + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the merge request. + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb new file mode 100644 index 00000000000..5437e32e9f1 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestsImporter + include ParallelScheduling + + def importer_class + PullRequestImporter + end + + def representation_class + Representation::PullRequest + end + + def sidekiq_worker_class + ImportPullRequestWorker + end + + def id_for_already_imported_cache(pr) + pr.number + end + + def each_object_to_import + super do |pr| + update_repository if update_repository?(pr) + yield pr + end + end + + def update_repository + # We set this column _before_ fetching the repository, and this is + # deliberate. If we were to update this column after the fetch we may + # miss out on changes pushed during the fetch or between the fetch and + # updating the timestamp. + project.update_column(:last_repository_updated_at, Time.zone.now) + + project.repository.fetch_remote('github', forced: false) + + pname = project.path_with_namespace + + Rails.logger + .info("GitHub importer finished updating repository for #{pname}") + + repository_updates_counter.increment(project: pname) + end + + def update_repository?(pr) + last_update = project.last_repository_updated_at || project.created_at + + return false if pr.updated_at < last_update + + # PRs may be updated without there actually being new commits, thus we + # check to make sure we only re-fetch if truly necessary. + !(commit_exists?(pr.head.sha) && commit_exists?(pr.base.sha)) + end + + def commit_exists?(sha) + project.repository.lookup(sha) + true + rescue Rugged::Error + false + end + + def collection_method + :pull_requests + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + + def repository_updates_counter + @repository_updates_counter ||= Gitlab::Metrics.counter( + :github_importer_repository_updates, + 'The number of times repositories have to be updated again' + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb new file mode 100644 index 00000000000..100f459fdcc --- /dev/null +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleasesImporter + include BulkImporting + + attr_reader :project, :client, :existing_tags + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + @existing_tags = project.releases.pluck(:tag).to_set + end + + def execute + bulk_insert(Release, build_releases) + end + + def build_releases + build_database_rows(each_release) + end + + def already_imported?(release) + existing_tags.include?(release.tag_name) + end + + def build(release) + { + tag: release.tag_name, + description: description_for(release), + created_at: release.created_at, + updated_at: release.updated_at, + project_id: project.id + } + end + + def each_release + client.releases(project.import_source) + end + + def description_for(release) + if release.body.present? + release.body + else + "Release for tag #{release.tag_name}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb new file mode 100644 index 00000000000..9cf2e7fd871 --- /dev/null +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class RepositoryImporter + include Gitlab::ShellAdapter + + attr_reader :project, :client + + def initialize(project, client) + @project = project + @client = client + end + + # Returns true if we should import the wiki for the project. + def import_wiki? + client.repository(project.import_source)&.has_wiki && + !project.wiki_repository_exists? + end + + # Imports the repository data. + # + # This method will return true if the data was imported successfully or + # the repository had already been imported before. + def execute + imported = + # It's possible a repository has already been imported when running + # this code, e.g. because we had to retry this job after + # `import_wiki?` raised a rate limit error. In this case we'll skip + # re-importing the main repository. + if project.repository.empty_repo? + import_repository + else + true + end + + update_clone_time if imported + + imported = import_wiki_repository if import_wiki? && imported + + imported + end + + def import_repository + project.ensure_repository + + refmap = Gitlab::GithubImport.refmap + project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true, remote_name: 'github') + + true + rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e + fail_import("Failed to import the repository: #{e.message}") + end + + def import_wiki_repository + wiki_path = "#{project.disk_path}.wiki" + wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') + storage_path = project.repository_storage_path + + gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) + + true + rescue Gitlab::Shell::Error => e + if e.message !~ /repository not exported/ + fail_import("Failed to import the wiki: #{e.message}") + else + true + end + end + + def update_clone_time + project.update_column(:last_repository_updated_at, Time.zone.now) + end + + def fail_import(message) + project.mark_import_as_failed(message) + false + end + end + end + end +end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb new file mode 100644 index 00000000000..211915f1d87 --- /dev/null +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # IssuableFinder can be used for caching and retrieving database IDs for + # issuable objects such as issues and pull requests. By caching these IDs we + # remove the need for running a lot of database queries when importing + # GitHub projects. + class IssuableFinder + attr_reader :project, :object + + # The base cache key to use for storing/retrieving issuable IDs. + CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}'.freeze + + # project - An instance of `Project`. + # object - The object to look up or set a database ID for. + def initialize(project, object) + @project = project + @object = object + end + + # Returns the database ID for the object. + # + # This method will return `nil` if no ID could be found. + def database_id + val = Caching.read(cache_key) + + val.to_i if val.present? + end + + # Associates the given database ID with the current object. + # + # database_id - The ID of the corresponding database row. + def cache_database_id(database_id) + Caching.write(cache_key, database_id) + end + + private + + def cache_key + CACHE_KEY % { + project: project.id, + type: cache_key_type, + iid: cache_key_iid + } + end + + # Returns the identifier to use for cache keys. + # + # For issues and pull requests this will be "Issue" or "MergeRequest" + # respectively. For diff notes this will return "MergeRequest", for + # regular notes it will either return "Issue" or "MergeRequest" depending + # on what type of object the note belongs to. + def cache_key_type + if object.respond_to?(:issuable_type) + object.issuable_type + elsif object.respond_to?(:noteable_type) + object.noteable_type + else + raise( + TypeError, + "Instances of #{object.class} are not supported" + ) + end + end + + def cache_key_iid + if object.respond_to?(:noteable_id) + object.noteable_id + elsif object.respond_to?(:iid) + object.iid + else + raise( + TypeError, + "Instances of #{object.class} are not supported" + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb new file mode 100644 index 00000000000..9be071141db --- /dev/null +++ b/lib/gitlab/github_import/label_finder.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class LabelFinder + attr_reader :project + + # The base cache key to use for storing/retrieving label IDs. + CACHE_KEY = 'github-import/label-finder/%{project}/%{name}'.freeze + + # project - An instance of `Project`. + def initialize(project) + @project = project + end + + # Returns the label ID for the given name. + def id_for(name) + Caching.read_integer(cache_key_for(name)) + end + + def build_cache + mapping = @project + .labels + .pluck(:id, :name) + .each_with_object({}) do |(id, name), hash| + hash[cache_key_for(name)] = id + end + + Caching.write_multiple(mapping) + end + + def cache_key_for(name) + CACHE_KEY % { project: project.id, name: name } + end + end + end +end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb new file mode 100644 index 00000000000..b25c4f7becf --- /dev/null +++ b/lib/gitlab/github_import/markdown_text.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class MarkdownText + attr_reader :text, :author, :exists + + def self.format(*args) + new(*args).to_s + end + + # text - The Markdown text as a String. + # author - An instance of `Gitlab::GithubImport::Representation::User` + # exists - Boolean that indicates the user exists in the GitLab database. + def initialize(text, author, exists = false) + @text = text + @author = author + @exists = exists + end + + def to_s + if exists + text + else + "*Created by: #{author.login}*\n\n#{text}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb new file mode 100644 index 00000000000..208d15dc144 --- /dev/null +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class MilestoneFinder + attr_reader :project + + # The base cache key to use for storing/retrieving milestone IDs. + CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}'.freeze + + # project - An instance of `Project` + def initialize(project) + @project = project + end + + # issuable - An instance of `Gitlab::GithubImport::Representation::Issue` + # or `Gitlab::GithubImport::Representation::PullRequest`. + def id_for(issuable) + return unless issuable.milestone_number + + Caching.read_integer(cache_key_for(issuable.milestone_number)) + end + + def build_cache + mapping = @project + .milestones + .pluck(:id, :iid) + .each_with_object({}) do |(id, iid), hash| + hash[cache_key_for(iid)] = id + end + + Caching.write_multiple(mapping) + end + + def cache_key_for(iid) + CACHE_KEY % { project: project.id, iid: iid } + end + end + end +end diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb new file mode 100644 index 00000000000..c3db2d0b469 --- /dev/null +++ b/lib/gitlab/github_import/page_counter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # PageCounter can be used to keep track of the last imported page of a + # collection, allowing workers to resume where they left off in the event of + # an error. + class PageCounter + attr_reader :cache_key + + # The base cache key to use for storing the last page number. + CACHE_KEY = 'github-importer/page-counter/%{project}/%{collection}'.freeze + + def initialize(project, collection) + @cache_key = CACHE_KEY % { project: project.id, collection: collection } + end + + # Sets the page number to the given value. + # + # Returns true if the page number was overwritten, false otherwise. + def set(page) + Caching.write_if_greater(cache_key, page) + end + + # Returns the current value from the cache. + def current + Caching.read_integer(cache_key) || 1 + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb new file mode 100644 index 00000000000..6da11e6ef08 --- /dev/null +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # The ParallelImporter schedules the importing of a GitHub project using + # Sidekiq. + class ParallelImporter + attr_reader :project + + def self.async? + true + end + + def self.imports_repository? + true + end + + def initialize(project) + @project = project + end + + def execute + jid = generate_jid + + # The original import JID is the JID of the RepositoryImportWorker job, + # which will be removed once that job completes. Reusing that JID could + # result in StuckImportJobsWorker marking the job as stuck before we get + # to running Stage::ImportRepositoryWorker. + # + # We work around this by setting the JID to a custom generated one, then + # refreshing it in the various stages whenever necessary. + Gitlab::SidekiqStatus + .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + + project.update_column(:import_jid, jid) + + Stage::ImportRepositoryWorker + .perform_async(project.id) + + true + end + + def generate_jid + "github-importer/#{project.id}" + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb new file mode 100644 index 00000000000..d4d1357f5a3 --- /dev/null +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module ParallelScheduling + attr_reader :project, :client, :page_counter, :already_imported_cache_key + + # The base cache key to use for tracking already imported objects. + ALREADY_IMPORTED_CACHE_KEY = + 'github-importer/already-imported/%{project}/%{collection}'.freeze + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + # parallel - When set to true the objects will be imported in parallel. + def initialize(project, client, parallel: true) + @project = project + @client = client + @parallel = parallel + @page_counter = PageCounter.new(project, collection_method) + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def parallel? + @parallel + end + + def execute + retval = + if parallel? + parallel_import + else + sequential_import + end + + # Once we have completed all work we can remove our "already exists" + # cache so we don't put too much pressure on Redis. + # + # We don't immediately remove it since it's technically possible for + # other instances of this job to still run, instead we set the + # expiration time to a lower value. This prevents the other jobs from + # still scheduling duplicates while. Since all work has already been + # completed those jobs will just cycle through any remaining pages while + # not scheduling anything. + Caching.expire(already_imported_cache_key, 15.minutes.to_i) + + retval + end + + # Imports all the objects in sequence in the current thread. + def sequential_import + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + importer_class.new(repr, project, client).execute + end + end + + # Imports all objects in parallel by scheduling a Sidekiq job for every + # individual object. + def parallel_import + waiter = JobWaiter.new + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + sidekiq_worker_class + .perform_async(project.id, repr.to_hash, waiter.key) + + waiter.jobs_remaining += 1 + end + + waiter + end + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + repo = project.import_source + + # We inject the page number here to make sure that all importers always + # start where they left off. Simply starting over wouldn't work for + # repositories with a lot of data (e.g. tens of thousands of comments). + options = collection_options.merge(page: page_counter.current) + + client.each_page(collection_method, repo, options) do |page| + # Technically it's possible that the same work is performed multiple + # times, as Sidekiq doesn't guarantee there will ever only be one + # instance of a job. In such a scenario it's possible for one job to + # have a lower page number (e.g. 5) compared to another (e.g. 10). In + # this case we skip over all the objects until we have caught up, + # reducing the number of duplicate jobs scheduled by the provided + # block. + next unless page_counter.set(page.number) + + page.objects.each do |object| + next if already_imported?(object) + + yield object + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(object) + end + end + end + + # Returns true if the given object has already been imported, false + # otherwise. + # + # object - The object to check. + def already_imported?(object) + id = id_for_already_imported_cache(object) + + Caching.set_includes?(already_imported_cache_key, id) + end + + # Marks the given object as "already imported". + def mark_as_imported(object) + id = id_for_already_imported_cache(object) + + Caching.set_add(already_imported_cache_key, id) + end + + # Returns the ID to use for the cache used for checking if an object has + # already been imported or not. + # + # object - The object we may want to import. + def id_for_already_imported_cache(object) + raise NotImplementedError + end + + # The class used for converting API responses to Hashes when performing + # the import. + def representation_class + raise NotImplementedError + end + + # The class to use for importing objects when importing them sequentially. + def importer_class + raise NotImplementedError + end + + # The Sidekiq worker class used for scheduling the importing of objects in + # parallel. + def sidekiq_worker_class + raise NotImplementedError + end + + # The name of the method to call to retrieve the data to import. + def collection_method + raise NotImplementedError + end + + # Any options to be passed to the method used for retrieving the data to + # import. + def collection_options + {} + end + end + end +end diff --git a/lib/gitlab/github_import/rate_limit_error.rb b/lib/gitlab/github_import/rate_limit_error.rb new file mode 100644 index 00000000000..cc2de909c29 --- /dev/null +++ b/lib/gitlab/github_import/rate_limit_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Error that will be raised when we're about to reach (or have reached) the + # GitHub API's rate limit. + RateLimitError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/github_import/representation.rb b/lib/gitlab/github_import/representation.rb new file mode 100644 index 00000000000..639477ef2a2 --- /dev/null +++ b/lib/gitlab/github_import/representation.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + TIMESTAMP_KEYS = %i[created_at updated_at merged_at].freeze + + # Converts a Hash with String based keys to one that can be used by the + # various Representation classes. + # + # Example: + # + # Representation.symbolize_hash('number' => 10) # => { number: 10 } + def self.symbolize_hash(raw_hash = nil) + hash = raw_hash.deep_symbolize_keys + + TIMESTAMP_KEYS.each do |key| + hash[key] = Time.parse(hash[key]) if hash[key].is_a?(String) + end + + hash + end + end + end +end diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb new file mode 100644 index 00000000000..bb7439a0641 --- /dev/null +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class DiffNote + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :noteable_type, :noteable_id, :commit_id, :file_path, + :diff_hunk, :author, :note, :created_at, :updated_at, + :github_id + + NOTEABLE_ID_REGEX = /\/pull\/(?<iid>\d+)/i + + # Builds a diff note from a GitHub API response. + # + # note - An instance of `Sawyer::Resource` containing the note details. + def self.from_api_response(note) + matches = note.html_url.match(NOTEABLE_ID_REGEX) + + unless matches + raise( + ArgumentError, + "The note URL #{note.html_url.inspect} is not supported" + ) + end + + user = Representation::User.from_api_response(note.user) if note.user + hash = { + noteable_type: 'MergeRequest', + noteable_id: matches[:iid].to_i, + file_path: note.path, + commit_id: note.commit_id, + diff_hunk: note.diff_hunk, + author: user, + note: note.body, + created_at: note.created_at, + updated_at: note.updated_at, + github_id: note.id + } + + new(hash) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + def line_code + diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last + + Gitlab::Git + .diff_line_code(file_path, diff_line.new_pos, diff_line.old_pos) + end + + # Returns a Hash that can be used to populate `notes.st_diff`, removing + # the need for requesting Git data for every diff note. + def diff_hash + { + diff: diff_hunk, + new_path: file_path, + old_path: file_path, + + # These fields are not displayed for LegacyDiffNote notes, so it + # doesn't really matter what we set them to. + a_mode: '100644', + b_mode: '100644', + new_file: false + } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/expose_attribute.rb b/lib/gitlab/github_import/representation/expose_attribute.rb new file mode 100644 index 00000000000..c3405759631 --- /dev/null +++ b/lib/gitlab/github_import/representation/expose_attribute.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module ExposeAttribute + extend ActiveSupport::Concern + + module ClassMethods + # Defines getter methods for the given attribute names. + # + # Example: + # + # expose_attribute :iid, :title + def expose_attribute(*names) + names.each do |name| + name = name.to_sym + + define_method(name) { attributes[name] } + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb new file mode 100644 index 00000000000..f3071b3e2b3 --- /dev/null +++ b/lib/gitlab/github_import/representation/issue.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Issue + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :iid, :title, :description, :milestone_number, + :created_at, :updated_at, :state, :assignees, + :label_names, :author + + # Builds an issue from a GitHub API response. + # + # issue - An instance of `Sawyer::Resource` containing the issue + # details. + def self.from_api_response(issue) + user = + if issue.user + Representation::User.from_api_response(issue.user) + end + + hash = { + iid: issue.number, + title: issue.title, + description: issue.body, + milestone_number: issue.milestone&.number, + state: issue.state == 'open' ? :opened : :closed, + assignees: issue.assignees.map do |u| + Representation::User.from_api_response(u) + end, + label_names: issue.labels.map(&:name), + author: user, + created_at: issue.created_at, + updated_at: issue.updated_at, + pull_request: issue.pull_request ? true : false + } + + new(hash) + end + + # Builds a new issue using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:state] = hash[:state].to_sym + hash[:assignees].map! { |u| Representation::User.from_json_hash(u) } + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A hash containing the raw issue details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def truncated_title + title.truncate(255) + end + + def labels? + label_names && label_names.any? + end + + def pull_request? + attributes[:pull_request] + end + + def issuable_type + pull_request? ? 'MergeRequest' : 'Issue' + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb new file mode 100644 index 00000000000..a68bc4c002f --- /dev/null +++ b/lib/gitlab/github_import/representation/note.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Note + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :noteable_id, :noteable_type, :author, :note, + :created_at, :updated_at, :github_id + + NOTEABLE_TYPE_REGEX = /\/(?<type>(pull|issues))\/(?<iid>\d+)/i + + # Builds a note from a GitHub API response. + # + # note - An instance of `Sawyer::Resource` containing the note details. + def self.from_api_response(note) + matches = note.html_url.match(NOTEABLE_TYPE_REGEX) + + if !matches || !matches[:type] + raise( + ArgumentError, + "The note URL #{note.html_url.inspect} is not supported" + ) + end + + noteable_type = + if matches[:type] == 'pull' + 'MergeRequest' + else + 'Issue' + end + + user = Representation::User.from_api_response(note.user) if note.user + hash = { + noteable_type: noteable_type, + noteable_id: matches[:iid].to_i, + author: user, + note: note.body, + created_at: note.created_at, + updated_at: note.updated_at, + github_id: note.id + } + + new(hash) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + alias_method :issuable_type, :noteable_type + end + end + end +end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb new file mode 100644 index 00000000000..593b491a837 --- /dev/null +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class PullRequest + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :iid, :title, :description, :source_branch, + :source_branch_sha, :target_branch, :target_branch_sha, + :milestone_number, :author, :assignee, :created_at, + :updated_at, :merged_at, :source_repository_id, + :target_repository_id, :source_repository_owner + + # Builds a PR from a GitHub API response. + # + # issue - An instance of `Sawyer::Resource` containing the PR details. + def self.from_api_response(pr) + assignee = + if pr.assignee + Representation::User.from_api_response(pr.assignee) + end + + user = Representation::User.from_api_response(pr.user) if pr.user + hash = { + iid: pr.number, + title: pr.title, + description: pr.body, + source_branch: pr.head.ref, + target_branch: pr.base.ref, + source_branch_sha: pr.head.sha, + target_branch_sha: pr.base.sha, + source_repository_id: pr.head&.repo&.id, + target_repository_id: pr.base&.repo&.id, + source_repository_owner: pr.head&.user&.login, + state: pr.state == 'open' ? :opened : :closed, + milestone_number: pr.milestone&.number, + author: user, + assignee: assignee, + created_at: pr.created_at, + updated_at: pr.updated_at, + merged_at: pr.merged_at + } + + new(hash) + end + + # Builds a new PR using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:state] = hash[:state].to_sym + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + # Assignees are optional so we only convert it from a Hash if one was + # set. + hash[:assignee] &&= Representation::User + .from_json_hash(hash[:assignee]) + + new(hash) + end + + # attributes - A Hash containing the raw PR details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def truncated_title + title.truncate(255) + end + + # Returns a formatted source branch. + # + # For cross-project pull requests the branch name will be in the format + # `owner-name:branch-name`. + def formatted_source_branch + if cross_project? && source_repository_owner + "#{source_repository_owner}:#{source_branch}" + elsif source_branch == target_branch + # Sometimes the source and target branch are the same, but GitLab + # doesn't support this. This can happen when both the user and + # source repository have been deleted, and the PR was submitted from + # the fork's master branch. + "#{source_branch}-#{iid}" + else + source_branch + end + end + + def state + if merged_at + :merged + else + attributes[:state] + end + end + + def cross_project? + return true unless source_repository_id + + source_repository_id != target_repository_id + end + + def issuable_type + 'MergeRequest' + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/to_hash.rb b/lib/gitlab/github_import/representation/to_hash.rb new file mode 100644 index 00000000000..4a0f36ab8f0 --- /dev/null +++ b/lib/gitlab/github_import/representation/to_hash.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module ToHash + # Converts the current representation to a Hash. The keys of this Hash + # will be Symbols. + def to_hash + hash = {} + + attributes.each do |key, value| + hash[key] = convert_value_for_to_hash(value) + end + + hash + end + + def convert_value_for_to_hash(value) + if value.is_a?(Array) + value.map { |v| convert_value_for_to_hash(v) } + elsif value.respond_to?(:to_hash) + value.to_hash + else + value + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb new file mode 100644 index 00000000000..e00dcfca33d --- /dev/null +++ b/lib/gitlab/github_import/representation/user.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class User + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :login + + # Builds a user from a GitHub API response. + # + # user - An instance of `Sawyer::Resource` containing the user details. + def self.from_api_response(user) + new(id: user.id, login: user.login) + end + + # Builds a user using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the user details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + end + end + end +end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb new file mode 100644 index 00000000000..4f7324536a0 --- /dev/null +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # The SequentialImporter imports a GitHub project in a single thread, + # without using Sidekiq. This makes it useful for testing purposes as well + # as Rake tasks, but it should be avoided for anything else in favour of the + # parallel importer. + class SequentialImporter + attr_reader :project, :client + + SEQUENTIAL_IMPORTERS = [ + Importer::LabelsImporter, + Importer::MilestonesImporter, + Importer::ReleasesImporter + ].freeze + + PARALLEL_IMPORTERS = [ + Importer::PullRequestsImporter, + Importer::IssuesImporter, + Importer::DiffNotesImporter, + Importer::NotesImporter + ].freeze + + # project - The project to import the data into. + # token - The token to use for the GitHub API. + def initialize(project, token: nil) + @project = project + @client = GithubImport + .new_client_for(project, token: token, parallel: false) + end + + def execute + Importer::RepositoryImporter.new(project, client).execute + + SEQUENTIAL_IMPORTERS.each do |klass| + klass.new(project, client).execute + end + + PARALLEL_IMPORTERS.each do |klass| + klass.new(project, client, parallel: false).execute + end + + project.repository.after_import + + true + end + end + end +end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb new file mode 100644 index 00000000000..be1259662a7 --- /dev/null +++ b/lib/gitlab/github_import/user_finder.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Class that can be used for finding a GitLab user ID based on a GitHub user + # ID or username. + # + # Any found user IDs are cached in Redis to reduce the number of SQL queries + # executed over time. Valid keys are refreshed upon access so frequently + # used keys stick around. + # + # Lookups are cached even if no ID was found to remove the need for querying + # the database when most queries are not going to return results anyway. + class UserFinder + attr_reader :project, :client + + # The base cache key to use for caching user IDs for a given GitHub user + # ID. + ID_CACHE_KEY = 'github-import/user-finder/user-id/%s'.freeze + + # The base cache key to use for caching user IDs for a given GitHub email + # address. + ID_FOR_EMAIL_CACHE_KEY = + 'github-import/user-finder/id-for-email/%s'.freeze + + # The base cache key to use for caching the Email addresses of GitHub + # usernames. + EMAIL_FOR_USERNAME_CACHE_KEY = + 'github-import/user-finder/email-for-username/%s'.freeze + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + end + + # Returns the GitLab user ID of an object's author. + # + # If the object has no author ID we'll use the ID of the GitLab ghost + # user. + def author_id_for(object) + id = + if object&.author + user_id_for(object.author) + else + GithubImport.ghost_user_id + end + + if id + [id, true] + else + [project.creator_id, false] + end + end + + # Returns the GitLab user ID of an issuable's assignee. + def assignee_id_for(issuable) + user_id_for(issuable.assignee) if issuable.assignee + end + + # Returns the GitLab user ID for a GitHub user. + # + # user - An instance of `Gitlab::GithubImport::Representation::User`. + def user_id_for(user) + find(user.id, user.login) + end + + # Returns the GitLab ID for the given GitHub ID or username. + # + # id - The ID of the GitHub user. + # username - The username of the GitHub user. + def find(id, username) + email = email_for_github_username(username) + cached, found_id = find_from_cache(id, email) + + return found_id if found_id + + # We only want to query the database if necessary. If previous lookups + # didn't yield a user ID we won't query the database again until the + # keys expire. + find_id_from_database(id, email) unless cached + end + + # Finds a user ID from the cache for a given GitHub ID or Email. + def find_from_cache(id, email = nil) + id_exists, id_for_github_id = cached_id_for_github_id(id) + + return [id_exists, id_for_github_id] if id_for_github_id + + # Just in case no Email address could be retrieved (for whatever reason) + return [false] unless email + + cached_id_for_github_email(email) + end + + # Finds a GitLab user ID from the database for a given GitHub user ID or + # Email. + def find_id_from_database(id, email) + id_for_github_id(id) || id_for_github_email(email) + end + + def email_for_github_username(username) + cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username + email = Caching.read(cache_key) + + unless email + user = client.user(username) + email = Caching.write(cache_key, user.email) if user + end + + email + end + + def cached_id_for_github_id(id) + read_id_from_cache(ID_CACHE_KEY % id) + end + + def cached_id_for_github_email(email) + read_id_from_cache(ID_FOR_EMAIL_CACHE_KEY % email) + end + + # Queries and caches the GitLab user ID for a GitHub user ID, if one was + # found. + def id_for_github_id(id) + gitlab_id = query_id_for_github_id(id) || nil + + Caching.write(ID_CACHE_KEY % id, gitlab_id) + end + + # Queries and caches the GitLab user ID for a GitHub email, if one was + # found. + def id_for_github_email(email) + gitlab_id = query_id_for_github_email(email) || nil + + Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) + end + + def query_id_for_github_id(id) + User.for_github_id(id).pluck(:id).first + end + + def query_id_for_github_email(email) + User.by_any_email(email).pluck(:id).first + end + + # Reads an ID from the cache. + # + # The return value is an Array with two values: + # + # 1. A boolean indicating if the key was present or not. + # 2. The ID as an Integer, or nil in case no ID could be found. + def read_id_from_cache(key) + value = Caching.read(key) + exists = !value.nil? + number = value.to_i + + # The cache key may be empty to indicate a previously looked up user for + # which we couldn't find an ID. + [exists, number.positive? ? number : nil] + end + end + end +end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index f1007daab5d..075b3982608 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -65,6 +65,7 @@ module Gitlab y << item end break if items.empty? || items.size < per_page + page += 1 end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3a666c2268b..dfcdfc307b6 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -20,7 +20,7 @@ module Gitlab gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') - gon.sprite_icons = ActionController::Base.helpers.asset_path('icons.svg') + gon.sprite_icons = IconsHelper.sprite_icon_path if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index de9cab80a02..e29dd0d5b0e 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -4,7 +4,6 @@ module Gitlab SAFE_HOOK_ATTRIBUTES = %i[ assignee_id author_id - branch_name closed_at confidential created_at @@ -29,6 +28,7 @@ module Gitlab SAFE_HOOK_RELATIONS = %i[ assignees labels + total_time_spent ].freeze attr_accessor :issue diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 503452c8ff3..ae9b68eb648 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -33,6 +33,7 @@ module Gitlab SAFE_HOOK_RELATIONS = %i[ assignee labels + total_time_spent ].freeze attr_accessor :merge_request diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 50ee879129c..2066005dddc 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.0'.freeze + VERSION = '0.2.1'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 263599831bf..f2b193c79cb 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -133,8 +133,6 @@ methods: - :type services: - :type - merge_request_diff: - - :utf8_st_diffs merge_request_diff_files: - :utf8_diff merge_requests: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index fbdd74788bc..c14646b0611 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,6 +1,10 @@ module Gitlab module ImportExport class Importer + def self.imports_repository? + true + end + def initialize(project) @archive_file = project.import_source @current_user = project.creator diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 61db4bd9ccc..f3d7407383c 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -1,7 +1,7 @@ module Gitlab module ImportExport class MergeRequestParser - FORKED_PROJECT_ID = -1 + FORKED_PROJECT_ID = nil def initialize(project, diff_head_sha, merge_request, relation_hash) @project = project diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 639f4f0c3f0..c518943be59 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -60,6 +60,8 @@ module Gitlab end end + @project.merge_requests.set_latest_merge_request_diff_ids! + @saved end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 2b34ceb5831..d7d1b05e8b9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -58,7 +58,6 @@ module Gitlab def setup_models case @relation_name - when :merge_request_diff then setup_st_diff_commits when :merge_request_diff_files then setup_diff when :notes then setup_note when :project_label, :project_labels then setup_label @@ -208,13 +207,6 @@ module Gitlab relation_class: relation_class) end - def setup_st_diff_commits - @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs') - - HashUtil.deep_symbolize_array!(@relation_hash['st_diffs']) - HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) - end - def setup_diff @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index f9ae5079d7c..627a487d577 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -24,8 +24,7 @@ module Gitlab end def uploads_path - # TODO: decide what to do with uploads. We will use UUIDs here too? - File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) + FileUploader.dynamic_path_segment(@project) end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 5404dc11a87..eeb03625479 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -8,14 +8,14 @@ module Gitlab ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ - ImportSource.new('github', 'GitHub', Github::Import), + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repo by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), - ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer) ].freeze class << self diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index 977c05910d3..0c9de72329c 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -1,6 +1,14 @@ module Gitlab module IssuableMetadata def issuable_meta_data(issuable_collection, collection_type) + # ActiveRecord uses Object#extend for null relations. + if !(issuable_collection.singleton_class < ActiveRecord::NullRelation) && + issuable_collection.respond_to?(:limit_value) && + issuable_collection.limit_value.nil? + + raise 'Collection must have a limit applied for preloading meta-data' + end + # map has to be used here since using pluck or select will # throw an error when ordering issuables by priority which inserts # a new order into the collection. diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 4d6bbda15f3..f654508c391 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -19,11 +19,13 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end - attr_reader :key, :jobs_remaining, :finished + attr_reader :key, :finished + attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for - def initialize(jobs_remaining) - @key = "gitlab:job_waiter:#{SecureRandom.uuid}" + # key - The key of this waiter. + def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}") + @key = key @jobs_remaining = jobs_remaining @finished = [] end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 7a50f07f3c5..407cdefc04d 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -18,7 +18,7 @@ module Gitlab def initialize(kubeclient) @kubeclient = kubeclient - @namespace = Namespace.new(NAMESPACE, kubeclient) + @namespace = Gitlab::Kubernetes::Namespace.new(NAMESPACE, kubeclient) end def install(command) diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb index c8479fbc0e8..fbbddb7bffa 100644 --- a/lib/gitlab/kubernetes/namespace.rb +++ b/lib/gitlab/kubernetes/namespace.rb @@ -12,6 +12,7 @@ module Gitlab @client.get_namespace(name) rescue ::KubeException => ke raise ke unless ke.error_code == 404 + false end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index ed1de73f8c6..7274d1c3b43 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -62,6 +62,7 @@ module Gitlab def user return nil unless ldap_user + Gitlab::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) end end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 4d5c67ed892..3945df27eed 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -9,11 +9,8 @@ module Gitlab class User < Gitlab::OAuth::User class << self def find_by_uid_and_provider(uid, provider) - uid = Gitlab::LDAP::Person.normalize_dn(uid) + identity = ::Identity.with_extern_uid(provider, uid).take - identity = ::Identity - .where(provider: provider) - .where(extern_uid: uid).last identity && identity.user end end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index f330041cc00..2f07fde406c 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class BaseFormatter attr_reader :client, :formatter, :project, :raw_data diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/legacy_github_import/branch_formatter.rb index 8aa885fb811..80fe1d67209 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/legacy_github_import/branch_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class BranchFormatter < BaseFormatter delegate :repo, :sha, :ref, to: :raw_data diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb new file mode 100644 index 00000000000..53c910d44bd --- /dev/null +++ b/lib/gitlab/legacy_github_import/client.rb @@ -0,0 +1,148 @@ +module Gitlab + module LegacyGithubImport + class Client + GITHUB_SAFE_REMAINING_REQUESTS = 100 + GITHUB_SAFE_SLEEP_TIME = 500 + + attr_reader :access_token, :host, :api_version + + def initialize(access_token, host: nil, api_version: 'v3') + @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version + @users = {} + + if access_token + ::Octokit.auto_paginate = false + end + end + + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: api_endpoint, + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end + + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' + end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) + end + + def authorize_url(redirect_uri) + client.auth_code.authorize_url({ + redirect_uri: redirect_uri, + scope: "repo, user, user:email" + }) + end + + def get_token(code) + client.auth_code.get_token(code).token + end + + def method_missing(method, *args, &block) + if api.respond_to?(method) + request(method, *args, &block) + else + super(method, *args, &block) + end + end + + def respond_to?(method) + api.respond_to?(method) || super + end + + def user(login) + return nil unless login.present? + return @users[login] if @users.key?(login) + + @users[login] = api.user(login) + end + + private + + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } + end + + def github_options + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end + end + + def rate_limit + api.rate_limit! + # GitHub Rate Limit API returns 404 when the rate limit is + # disabled. In this case we just want to return gracefully + # instead of spitting out an error. + rescue Octokit::NotFound + nil + end + + def has_rate_limit? + return @has_rate_limit if defined?(@has_rate_limit) + + @has_rate_limit = rate_limit.present? + end + + def rate_limit_exceed? + has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + end + + def rate_limit_sleep_time + rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + end + + def request(method, *args, &block) + sleep rate_limit_sleep_time if rate_limit_exceed? + + data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend + return data unless data.is_a?(Array) + + last_response = api.last_response + + if block_given? + yield data + # api.last_response could change while we're yielding (e.g. fetching labels for each PR) + # so we cache our own last response + each_response_page(last_response, &block) + else + each_response_page(last_response) { |page| data.concat(page) } + data + end + end + + def each_response_page(last_response) + while last_response.rels[:next] + sleep rate_limit_sleep_time if rate_limit_exceed? + last_response = last_response.rels[:next].get + yield last_response.data if last_response.data.is_a?(Array) + end + end + end + end +end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb index 8911b81ec9a..d2c7a8ae9f4 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class CommentFormatter < BaseFormatter attr_writer :author_id diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index b8c07460ebb..0526ef9eb13 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -1,8 +1,12 @@ module Gitlab - module GithubImport + module LegacyGithubImport class Importer include Gitlab::ShellAdapter + def self.refmap + Gitlab::GithubImport.refmap + end + attr_reader :errors, :project, :repo, :repo_url def initialize(project) @@ -15,6 +19,7 @@ module Gitlab def client return @client if defined?(@client) + unless credentials raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index 27b171d6ddb..de55382d3ad 100644 --- a/lib/gitlab/github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class IssuableFormatter < BaseFormatter attr_writer :assignee_id, :author_id diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb index 977cd0423ba..4c8825ccf19 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class IssueFormatter < IssuableFormatter def attributes { diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index 211ccdc51bb..c3eed12e739 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class LabelFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb index dd782eff059..a565294384d 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class MilestoneFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index a55adc9b1c8..41e7eac4d08 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class ProjectCreator include Gitlab::CurrentSettings diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb index 150afa31432..94c2e99066a 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class PullRequestFormatter < IssuableFormatter delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index 1ad702a6058..3ed9d4f76da 100644 --- a/lib/gitlab/github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class ReleaseFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 04c2964da20..6d8055622f1 100644 --- a/lib/gitlab/github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class UserFormatter attr_reader :client, :raw diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/legacy_github_import/wiki_formatter.rb index ca8d96f5650..27f45875c7c 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/legacy_github_import/wiki_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class WikiFormatter attr_reader :project diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 8e57ba831c5..ead5d566871 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -27,6 +27,10 @@ module Gitlab end end + def deploy_key_pushable?(project) + actor.is_a?(DeployKey) && actor.can_push_to?(project) + end + def user? actor.is_a?(User) end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 90235095306..65d55576ac2 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -6,29 +6,15 @@ module Gitlab BASE_LABELS = { module: nil, method: nil }.freeze attr_reader :real_time, :cpu_time, :call_count, :labels - def self.call_real_duration_histogram - return @call_real_duration_histogram if @call_real_duration_histogram - - MUTEX.synchronize do - @call_real_duration_histogram ||= Gitlab::Metrics.histogram( - :gitlab_method_call_real_duration_seconds, - 'Method calls real duration', - Transaction::BASE_LABELS.merge(BASE_LABELS), - [0.1, 0.2, 0.5, 1, 2, 5, 10] - ) - end - end - - def self.call_cpu_duration_histogram - return @call_cpu_duration_histogram if @call_cpu_duration_histogram + def self.call_duration_histogram + return @call_duration_histogram if @call_duration_histogram MUTEX.synchronize do @call_duration_histogram ||= Gitlab::Metrics.histogram( - :gitlab_method_call_cpu_duration_seconds, - 'Method calls cpu duration', + :gitlab_method_call_duration_seconds, + 'Method calls real duration', Transaction::BASE_LABELS.merge(BASE_LABELS), - [0.1, 0.2, 0.5, 1, 2, 5, 10] - ) + [0.01, 0.05, 0.1, 0.5, 1]) end end @@ -59,8 +45,9 @@ module Gitlab @cpu_time += cpu_time @call_count += 1 - self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) - self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0) + if call_measurement_enabled? && above_threshold? + self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) + end retval end @@ -83,6 +70,10 @@ module Gitlab def above_threshold? real_time >= Metrics.method_call_threshold end + + def call_measurement_enabled? + Feature.get(:prometheus_metrics_method_instrumentation).enabled? + end end end end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 8b5a60e6b8b..f4901be9581 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -32,7 +32,7 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', {}) + metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) GC.stat.keys.each do |key| metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) @@ -96,12 +96,13 @@ module Gitlab def worker_label return {} unless defined?(Unicorn::Worker) + worker_no = ::Prometheus::Client::Support::Unicorn.worker_id if worker_no - { unicorn: worker_no } + { worker: worker_no } else - { unicorn: 'master' } + { worker: 'master' } end end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index 55c707d5386..df4bdf16847 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -11,8 +11,6 @@ module Gitlab # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } - - worker.metrics_tags.each { |tag, value| trans.add_tag(tag, value) } if worker.respond_to?(:metrics_tags) rescue Exception => error # rubocop: disable Lint/RescueException trans.add_event(:sidekiq_exception) diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 064299f40c8..ead1acb8d44 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -7,6 +7,7 @@ module Gitlab def sql(event) return unless current_transaction + metric_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) current_transaction.increment(:sql_duration, event.duration, false) diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index cfc6b2a2029..c6a56277922 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -42,12 +42,11 @@ module Gitlab project_url = URI.join(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - repository_url = case current_application_settings.enabled_git_access_protocol - when 'ssh' + repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh' shell = config.gitlab_shell port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" - when 'http', nil + else "#{project_url}.git" end @@ -66,6 +65,7 @@ module Gitlab project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX) return unless project_path_match + path = project_path_match[1] # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`. diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 8853dfa3d2d..c26656704d7 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -58,7 +58,7 @@ module Gitlab end def last_visited_url - @env['HTTP_REFERER'] || rack_session['user_return_to'] || Rails.application.routes.url_helpers.root_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url end def route_hash @@ -66,11 +66,7 @@ module Gitlab end def whitelisted_routes - logout_route || grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def logout_route - route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy' + grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end def sidekiq_route @@ -78,10 +74,16 @@ module Gitlab end def grack_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('.git/git-upload-pack') + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' end def lfs_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('/info/lfs/objects/batch') + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' end end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index eb3c9002710..c22d0a84860 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -55,7 +55,9 @@ module Gitlab def first_collection_last_page_size return @first_collection_last_page_size if defined?(@first_collection_last_page_size) - @first_collection_last_page_size = paginated_first_collection(first_collection_page_count).count + @first_collection_last_page_size = paginated_first_collection(first_collection_page_count) + .except(:select) + .size end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index b4b3b00c84d..552133234a3 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -157,7 +157,7 @@ module Gitlab end def find_by_uid_and_provider - identity = Identity.find_by(provider: auth_hash.provider, extern_uid: auth_hash.uid) + identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take identity && identity.user end diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 962ff4d3985..1d9a5d1a20a 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -11,6 +11,7 @@ module Gitlab rescue ActiveRecord::StaleObjectError retries -= 1 raise unless retries >= 0 + subject.reload end end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 9a91f8bf96a..7e5dfd33502 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -51,7 +51,6 @@ module Gitlab slash-command-logo.png snippets u - unicorn_test unsubscribes uploads users diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 561aa9e162c..e2662fc362b 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -47,8 +47,11 @@ module Gitlab startline = 0 result.each_line.each_with_index do |line, index| - if line =~ /^.*:.*:\d+:/ - ref, filename, startline = line.split(':') + matches = line.match(/^(?<ref>[^:]*):(?<filename>.*):(?<startline>\d+):/) + if matches + ref = matches[:ref] + filename = matches[:filename] + startline = matches[:startline] startline = startline.to_i - index extname = Regexp.escape(File.extname(filename)) basename = filename.sub(/#{extname}$/, '') diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 7ac6162b54d..5cddc96a643 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -76,7 +76,7 @@ module Gitlab timeframe_start: timeframe_start, timeframe_end: timeframe_end, ci_environment_slug: environment.slug, - kube_namespace: environment.project.kubernetes_service&.actual_namespace || '', + kube_namespace: environment.project.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} } end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index bc836dcc08d..9ff82d628c0 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,7 +1,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user).freeze + REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user epic).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index bd677ec4bf3..2c7b8af83f2 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -25,7 +25,7 @@ module Gitlab # See https://github.com/docker/distribution/blob/master/reference/regexp.go. # def container_repository_name_regex - @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} + @container_repository_regex ||= %r{\A[a-z0-9]+((?:[._/]|__|[-])[a-z0-9]+)*\Z} end ## diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 910533076b0..2c994536060 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -46,10 +46,10 @@ module Gitlab # Only replace the last occurence of `path`. # # `request.fullpath` includes the querystring - path = request.path.sub(%r{/#{path}/*(?!.*#{path})}, "/-/#{path}/") - path << "?#{request.query_string}" if request.query_string.present? + new_path = request.path.sub(%r{/#{path}(/*)(?!.*#{path})}, "/-/#{path}\\1") + new_path << "?#{request.query_string}" if request.query_string.present? - path + new_path end paths.each do |path| diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index e0a9d1dee77..d8faf7aad8c 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -28,6 +28,7 @@ module Gitlab def changed? return true unless gl_user + gl_user.changed? || gl_user.identities.any?(&:changed?) end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index efe8095beea..fef9d3e31d4 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -30,7 +30,7 @@ module Gitlab def initialize(current_user, limit_projects, query) @current_user = current_user @limit_projects = limit_projects || Project.all - @query = Shellwords.shellescape(query) if query.present? + @query = query end def objects(scope, page = nil) diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index f9ab9bd466f..30df7e4a831 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -8,7 +8,8 @@ end module Gitlab class Seeder def self.quiet - mute_mailer + mute_mailer unless Rails.env.test? + SeedFu.quiet = true yield diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index a37112ae5c4..a22a63665be 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -101,8 +101,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) - # Timeout should be less than 900 ideally, to prevent the memory killer - # to silently kill the process without knowing we are timing out here. + # The timeout ensures the subprocess won't hang forever cmd = [gitlab_shell_projects_path, 'import-project', storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"] gitlab_shell_fast_execute_raise_error(cmd) @@ -144,20 +143,27 @@ module Gitlab storage, "#{path}.git", "#{new_path}.git"]) end - # Fork repository to new namespace + # Fork repository to new path # forked_from_storage - forked-from project's storage path - # path - project path with namespace + # forked_from_disk_path - project disk path # forked_to_storage - forked-to project's storage path - # fork_namespace - namespace for forked project + # forked_to_disk_path - forked project disk path # # Ex. - # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") + # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") # # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. - def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) - gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', - forked_from_storage, "#{path}.git", forked_to_storage, - fork_namespace]) + def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) + gitlab_shell_fast_execute( + [ + gitlab_shell_projects_path, + 'fork-repository', + forked_from_storage, + "#{forked_from_disk_path}.git", + forked_to_storage, + "#{forked_to_disk_path}.git" + ] + ) end # Remove repository from file system @@ -368,6 +374,7 @@ module Gitlab output, status = gitlab_shell_fast_execute_helper(cmd, vars) raise Error, output unless status.zero? + true end diff --git a/lib/gitlab/shell_adapter.rb b/lib/gitlab/shell_adapter.rb index fbe2a7a0d72..053dd4ab9e0 100644 --- a/lib/gitlab/shell_adapter.rb +++ b/lib/gitlab/shell_adapter.rb @@ -5,7 +5,7 @@ module Gitlab module ShellAdapter def gitlab_shell - Gitlab::Shell.new + @gitlab_shell ||= Gitlab::Shell.new end end end diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb new file mode 100644 index 00000000000..dc9886732b5 --- /dev/null +++ b/lib/gitlab/sidekiq_config.rb @@ -0,0 +1,50 @@ +require 'yaml' + +module Gitlab + module SidekiqConfig + def self.redis_queues + @redis_queues ||= Sidekiq::Queue.all.map(&:name) + end + + # This method is called by `bin/sidekiq-cluster` in EE, which runs outside + # of bundler/Rails context, so we cannot use any gem or Rails methods. + def self.config_queues(rails_path = Rails.root.to_s) + @config_queues ||= begin + config = YAML.load_file(File.join(rails_path, 'config', 'sidekiq_queues.yml')) + config[:queues].map(&:first) + end + end + + def self.cron_workers + @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize } + end + + def self.workers + @workers ||= find_workers(Rails.root.join('app', 'workers')) + end + + def self.default_queues + [ActionMailer::DeliveryJob.queue_name, 'default'] + end + + def self.worker_queues + @worker_queues ||= (workers.map(&:queue) + default_queues).uniq + end + + def self.find_workers(root) + concerns = root.join('concerns').to_s + + workers = Dir[root.join('**', '*.rb')] + .reject { |path| path.start_with?(concerns) } + + workers.map! do |path| + ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') + + ns.camelize.constantize + end + + # Skip concerns + workers.select { |w| w < Sidekiq::Worker } + end + end +end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 7c2d1d8f887..5f0c98cb5a4 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -4,9 +4,15 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_WORD = /(?<=^| )"[^"]+"(?= |$)/ + REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/ class_methods do + def fuzzy_search(query, columns) + matches = columns.map { |col| fuzzy_arel_match(col, query) }.compact.reduce(:or) + + where(matches) + end + def to_pattern(query) if partial_matching?(query) "%#{sanitize_sql_like(query)}%" @@ -19,12 +25,19 @@ module Gitlab query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end - def to_fuzzy_arel(column, query) - words = select_fuzzy_words(query) + def fuzzy_arel_match(column, query) + query = query.squish + return nil unless query.present? - matches = words.map { |word| arel_table[column].matches(to_pattern(word)) } + words = select_fuzzy_words(query) - matches.reduce { |result, match| result.and(match) } + if words.any? + words.map { |word| arel_table[column].matches(to_pattern(word)) }.reduce(:and) + else + # No words of at least 3 chars, but we can search for an exact + # case insensitive match with the query as a whole + arel_table[column].matches(sanitize_sql_like(query)) + end end def select_fuzzy_words(query) @@ -32,7 +45,7 @@ module Gitlab query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } - words = query.split(/\s+/) + words = query.split quoted_words.map! { |quoted_word| quoted_word[1..-2] } diff --git a/lib/gitlab/storage_check.rb b/lib/gitlab/storage_check.rb new file mode 100644 index 00000000000..fe81513c9ec --- /dev/null +++ b/lib/gitlab/storage_check.rb @@ -0,0 +1,11 @@ +require_relative 'storage_check/cli' +require_relative 'storage_check/gitlab_caller' +require_relative 'storage_check/option_parser' +require_relative 'storage_check/response' + +module Gitlab + module StorageCheck + ENDPOINT = '/-/storage_check'.freeze + Options = Struct.new(:target, :token, :interval, :dryrun) + end +end diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb new file mode 100644 index 00000000000..04bf1bf1d26 --- /dev/null +++ b/lib/gitlab/storage_check/cli.rb @@ -0,0 +1,69 @@ +module Gitlab + module StorageCheck + class CLI + def self.start!(args) + runner = new(Gitlab::StorageCheck::OptionParser.parse!(args)) + runner.start_loop + end + + attr_reader :logger, :options + + def initialize(options) + @options = options + @logger = Logger.new(STDOUT) + end + + def start_loop + logger.info "Checking #{options.target} every #{options.interval} seconds" + + if options.dryrun + logger.info "Dryrun, exiting..." + return + end + + begin + loop do + response = GitlabCaller.new(options).call! + log_response(response) + update_settings(response) + + sleep options.interval + end + rescue Interrupt + logger.info "Ending storage-check" + end + end + + def update_settings(response) + previous_interval = options.interval + + if response.valid? + options.interval = response.check_interval || previous_interval + end + + if previous_interval != options.interval + logger.info "Interval changed: #{options.interval} seconds" + end + end + + def log_response(response) + unless response.valid? + return logger.error("Invalid response checking nfs storage: #{response.http_response.inspect}") + end + + if response.responsive_shards.any? + logger.debug("Responsive shards: #{response.responsive_shards.join(', ')}") + end + + warnings = [] + if response.skipped_shards.any? + warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" + end + if response.failing_shards.any? + warnings << "Failing shards: #{response.failing_shards.join(', ')}" + end + logger.warn(warnings.join(' - ')) if warnings.any? + end + end + end +end diff --git a/lib/gitlab/storage_check/gitlab_caller.rb b/lib/gitlab/storage_check/gitlab_caller.rb new file mode 100644 index 00000000000..44952b68844 --- /dev/null +++ b/lib/gitlab/storage_check/gitlab_caller.rb @@ -0,0 +1,39 @@ +require 'excon' + +module Gitlab + module StorageCheck + class GitlabCaller + def initialize(options) + @options = options + end + + def call! + Gitlab::StorageCheck::Response.new(get_response) + rescue Errno::ECONNREFUSED, Excon::Error + # Server not ready, treated as invalid response. + Gitlab::StorageCheck::Response.new(nil) + end + + def get_response + scheme, *other_parts = URI.split(@options.target) + socket_path = if scheme == 'unix' + other_parts.compact.join + end + + connection = Excon.new(@options.target, socket: socket_path) + connection.post(path: Gitlab::StorageCheck::ENDPOINT, + headers: headers) + end + + def headers + @headers ||= begin + headers = {} + headers['Content-Type'] = headers['Accept'] = 'application/json' + headers['TOKEN'] = @options.token if @options.token + + headers + end + end + end + end +end diff --git a/lib/gitlab/storage_check/option_parser.rb b/lib/gitlab/storage_check/option_parser.rb new file mode 100644 index 00000000000..66ed7906f97 --- /dev/null +++ b/lib/gitlab/storage_check/option_parser.rb @@ -0,0 +1,39 @@ +module Gitlab + module StorageCheck + class OptionParser + def self.parse!(args) + # Start out with some defaults + options = Gitlab::StorageCheck::Options.new(nil, nil, 1, false) + + parser = ::OptionParser.new do |opts| + opts.banner = "Usage: bin/storage_check [options]" + + opts.on('-t=string', '--target string', 'URL or socket to trigger storage check') do |value| + options.target = value + end + + opts.on('-T=string', '--token string', 'Health token to use') { |value| options.token = value } + + opts.on('-i=n', '--interval n', ::OptionParser::DecimalInteger, 'Seconds between checks') do |value| + options.interval = value + end + + opts.on('-d', '--dryrun', "Output what will be performed, but don't start the process") do |value| + options.dryrun = value + end + end + parser.parse!(args) + + unless options.target + raise ::OptionParser::InvalidArgument.new('Provide a URI to provide checks') + end + + if URI.parse(options.target).scheme.nil? + raise ::OptionParser::InvalidArgument.new('Add the scheme to the target, `unix://`, `https://` or `http://` are supported') + end + + options + end + end + end +end diff --git a/lib/gitlab/storage_check/response.rb b/lib/gitlab/storage_check/response.rb new file mode 100644 index 00000000000..326ab236e3e --- /dev/null +++ b/lib/gitlab/storage_check/response.rb @@ -0,0 +1,77 @@ +require 'json' + +module Gitlab + module StorageCheck + class Response + attr_reader :http_response + + def initialize(http_response) + @http_response = http_response + end + + def valid? + @http_response && (200...299).cover?(@http_response.status) && + @http_response.headers['Content-Type'].include?('application/json') && + parsed_response + end + + def check_interval + return nil unless parsed_response + + parsed_response['check_interval'] + end + + def responsive_shards + divided_results[:responsive_shards] + end + + def skipped_shards + divided_results[:skipped_shards] + end + + def failing_shards + divided_results[:failing_shards] + end + + private + + def results + return [] unless parsed_response + + parsed_response['results'] + end + + def divided_results + return @divided_results if @divided_results + + @divided_results = {} + @divided_results[:responsive_shards] = [] + @divided_results[:skipped_shards] = [] + @divided_results[:failing_shards] = [] + + results.each do |info| + name = info['storage'] + + case info['success'] + when true + @divided_results[:responsive_shards] << name + when false + @divided_results[:failing_shards] << name + else + @divided_results[:skipped_shards] << name + end + end + + @divided_results + end + + def parsed_response + return @parsed_response if defined?(@parsed_response) + + @parsed_response = JSON.parse(@http_response.body) + rescue JSON::JSONError + @parsed_response = nil + end + end + end +end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 11aeec1ebfa..f9faa134206 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -90,6 +90,7 @@ module Gitlab # Takes an array of integers, and returns an array of ranges covering the same integers def collapse_ranges(positions) return [] if positions.empty? + ranges = [] start = prev = positions[0] diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index cb7957e2af9..33f07fa0120 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -18,6 +18,7 @@ module Gitlab def read(path) blob = @repository.blob_at(@commit.id, path) if @commit raise FileNotFoundError if blob.nil? + blob.data end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index fee1a127fd7..13150ddab67 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -22,10 +22,12 @@ module Gitlab return true if blocked_user_or_hostname?(uri.user) return true if blocked_user_or_hostname?(uri.hostname) - server_ips = Resolv.getaddresses(uri.hostname) + server_ips = Addrinfo.getaddrinfo(uri.hostname, 80, nil, :STREAM).map(&:ip_address) return true if (blocked_ips & server_ips).any? rescue Addressable::URI::InvalidURIError return true + rescue SocketError + return false end false diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 1caa791c1be..59331c827af 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -70,6 +70,7 @@ module Gitlab def generate_full_url return @url unless valid_credentials? + @full_url = @url.dup @full_url.password = credentials[:password] if credentials[:password].present? diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 112d4939582..2adcc9809b3 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -79,7 +79,7 @@ module Gitlab def features_usage_data_ce { - signup: current_application_settings.signup_enabled?, + signup: current_application_settings.allow_signup?, ldap: Gitlab.config.ldap.enabled, gravatar: current_application_settings.gravatar_enabled?, omniauth: Gitlab.config.omniauth.enabled, diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index abb3d3a02c3..b3baaf036d8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -46,5 +46,22 @@ module Gitlab def random_string Random.rand(Float::MAX.to_i).to_s(36) end + + # See: http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby + # Cross-platform way of finding an executable in the $PATH. + # + # which('ruby') #=> /usr/bin/ruby + def which(cmd, env = ENV) + exts = env['PATHEXT'] ? env['PATHEXT'].split(';') : [''] + + env['PATH'].split(File::PATH_SEPARATOR).each do |path| + exts.each do |ext| + exe = File.join(path, "#{cmd}#{ext}") + return exe if File.executable?(exe) && !File.directory?(exe) + end + end + + nil + end end end diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb new file mode 100644 index 00000000000..a2ac9285b56 --- /dev/null +++ b/lib/gitlab/utils/strong_memoize.rb @@ -0,0 +1,31 @@ +module Gitlab + module Utils + module StrongMemoize + # Instead of writing patterns like this: + # + # def trigger_from_token + # return @trigger if defined?(@trigger) + # + # @trigger = Ci::Trigger.find_by_token(params[:token].to_s) + # end + # + # We could write it like: + # + # def trigger_from_token + # strong_memoize(:trigger) do + # Ci::Trigger.find_by_token(params[:token].to_s) + # end + # end + # + def strong_memoize(name) + ivar_name = "@#{name}" + + if instance_variable_defined?(ivar_name) + instance_variable_get(ivar_name) + else + instance_variable_set(ivar_name, yield) + end + end + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index c60bd91ea6e..11472ce6cce 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -99,6 +99,7 @@ module Gitlab def level_value(level) return level.to_i if level.to_i.to_s == level.to_s && string_options.key(level.to_i) + string_options[level] || PRIVATE end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e1219df1b25..5ab6cd5a4ef 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -58,7 +58,7 @@ module Gitlab end def artifact_upload_ok - { TempPath: ArtifactUploader.artifacts_upload_path } + { TempPath: JobArtifactUploader.artifacts_upload_path } end def send_git_blob(repository, blob) @@ -174,6 +174,7 @@ module Gitlab @secret ||= begin bytes = Base64.strict_decode64(File.read(secret_path).chomp) raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH + bytes end end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 9242cbe840c..b0563fb2d69 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -44,7 +44,7 @@ module GoogleApi service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token - service.get_zone_cluster(project_id, zone, cluster_id) + service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header) end def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:) @@ -62,14 +62,14 @@ module GoogleApi } } ) - service.create_cluster(project_id, zone, request_body) + service.create_cluster(project_id, zone, request_body, options: user_agent_header) end def projects_zones_operations(project_id, zone, operation_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token - service.get_zone_operation(project_id, zone, operation_id) + service.get_zone_operation(project_id, zone, operation_id, options: user_agent_header) end def parse_operation_id(self_link) @@ -82,6 +82,12 @@ module GoogleApi def token_life_time(expires_at) DateTime.strptime(expires_at, '%s').to_time.utc - Time.now.utc end + + def user_agent_header + Google::Apis::RequestOptions.new.tap do |options| + options.header = { 'User-Agent': "GitLab/#{Gitlab::VERSION.match('(\d+\.\d+)').captures.first} (GPN:GitLab;)" } + end + end end end end diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb index 05668c69006..f5485eb89fa 100644 --- a/lib/haml_lint/inline_javascript.rb +++ b/lib/haml_lint/inline_javascript.rb @@ -9,6 +9,7 @@ unless Rails.env.production? def visit_filter(node) return unless node.filter_type == 'javascript' + record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') end end diff --git a/lib/milestone_array.rb b/lib/milestone_array.rb new file mode 100644 index 00000000000..4ed8485b36a --- /dev/null +++ b/lib/milestone_array.rb @@ -0,0 +1,40 @@ +module MilestoneArray + class << self + def sort(array, sort_method) + case sort_method + when 'due_date_asc' + sort_asc_nulls_last(array, 'due_date') + when 'due_date_desc' + sort_desc_nulls_last(array, 'due_date') + when 'start_date_asc' + sort_asc_nulls_last(array, 'start_date') + when 'start_date_desc' + sort_desc_nulls_last(array, 'start_date') + when 'name_asc' + sort_asc(array, 'title') + when 'name_desc' + sort_asc(array, 'title').reverse + else + array + end + end + + private + + def sort_asc_nulls_last(array, attribute) + attribute = attribute.to_sym + + array.select(&attribute).sort_by(&attribute) + array.reject(&attribute) + end + + def sort_desc_nulls_last(array, attribute) + attribute = attribute.to_sym + + array.select(&attribute).sort_by(&attribute).reverse + array.reject(&attribute) + end + + def sort_asc(array, attribute) + array.sort_by(&attribute.to_sym) + end + end +end diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb deleted file mode 100644 index 939b23a3421..00000000000 --- a/lib/rouge/lexers/math.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Rouge - module Lexers - class Math < PlainText - title "A passthrough lexer used for LaTeX input" - desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter" - tag 'math' - end - end -end diff --git a/lib/rouge/lexers/plantuml.rb b/lib/rouge/lexers/plantuml.rb deleted file mode 100644 index 63c461764fc..00000000000 --- a/lib/rouge/lexers/plantuml.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Rouge - module Lexers - class Plantuml < PlainText - title "A passthrough lexer used for PlantUML input" - desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter" - tag 'plantuml' - end - end -end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 00221f77cf4..8b145fb4511 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -24,6 +24,7 @@ module SystemCheck # @param [BaseCheck] check class def <<(check) raise ArgumentError unless check.is_a?(Class) && check < BaseCheck + @checks << check end diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 99b3168d9eb..2301ec9b228 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 87ca39b079b..c2d3a6b6950 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -1,5 +1,28 @@ namespace :gemojione do desc 'Generates Emoji SHA256 digests' + + task aliases: ['yarn:check', 'environment'] do + require 'json' + + aliases = {} + + index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json') + index = JSON.parse(File.read(index_file)) + + index.each_pair do |key, data| + data['aliases'].each do |a| + a.tr!(':', '') + + aliases[a] = key + end + end + + out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') + File.open(out, 'w') do |handle| + handle.write(JSON.pretty_generate(aliases, indent: ' ', space: '', space_before: '')) + end + end + task digests: ['yarn:check', 'environment'] do require 'digest/sha2' require 'json' @@ -16,8 +39,13 @@ namespace :gemojione do fpath = File.join(dir, "#{emoji_hash['unicode']}.png") hash_digest = Digest::SHA256.file(fpath).hexdigest + category = emoji_hash['category'] + if name == 'gay_pride_flag' + category = 'flags' + end + entry = { - category: emoji_hash['category'], + category: category, moji: emoji_hash['moji'], description: emoji_hash['description'], unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), @@ -29,7 +57,6 @@ namespace :gemojione do end out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') - File.open(out, 'w') do |handle| handle.write(JSON.pretty_generate(resultant_emoji_map)) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 8ae1b6a626a..eb0f757aea7 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,11 +1,14 @@ namespace :gitlab do namespace :cleanup do + HASHED_REPOSITORY_NAME = '@hashed'.freeze + desc "GitLab | Cleanup | Clean namespaces" task dirs: :environment do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) + namespaces = Namespace.pluck(:path) + namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| git_base_path = repository_storage['path'] all_dirs = Dir.glob(git_base_path + '/*') @@ -59,7 +62,11 @@ namespace :gitlab do .sub(%r{^/*}, '') .chomp('.git') .chomp('.wiki') - next if Project.find_by_full_path(repo_with_namespace) + + # TODO ignoring hashed repositories for now. But revisit to fully support + # possible orphaned hashed repos + next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace) + new_path = path + move_suffix puts path.inspect + ' -> ' + new_path.inspect File.rename(path, new_path) @@ -75,6 +82,7 @@ namespace :gitlab do User.find_each do |user| next unless user.ldap_user? + print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." if Gitlab::LDAP::Access.allowed?(user) puts " [OK]".color(:green) diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 8377fe3269d..4d880c05f99 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -14,18 +14,18 @@ namespace :gitlab do checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) + command = %w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + _, status = Gitlab::Popen.popen(%w[which gmake]) - command = status.zero? ? ['gmake'] : ['make'] + command << (status.zero? ? 'gmake' : 'make') - if Rails.env.test? - command += %W[BUNDLE_PATH=#{Bundler.bundle_path}] - end + command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? Dir.chdir(args.dir) do create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? - Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + command) } + Bundler.with_original_env { run_command!(command) } end end end @@ -78,13 +78,18 @@ namespace :gitlab do config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } + config[:bin_dir] = Gitlab.config.gitaly.client_path + TOML.dump(config) end def create_gitaly_configuration - File.open("config.toml", "w") do |f| + File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f| f.puts gitaly_configuration_toml end + rescue Errno::EEXIST + puts "Skipping config.toml generation:" + puts "A configuration file already exists." rescue ArgumentError => e puts "Skipping config.toml generation:" puts e.message diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake deleted file mode 100644 index 6cbc83b8973..00000000000 --- a/lib/tasks/gitlab/sidekiq.rake +++ /dev/null @@ -1,47 +0,0 @@ -namespace :gitlab do - namespace :sidekiq do - QUEUE = 'queue:post_receive'.freeze - - desc 'Drop all Sidekiq PostReceive jobs for a given project' - task :drop_post_receive, [:project] => :environment do |t, args| - unless args.project.present? - abort "Please specify the project you want to drop PostReceive jobs for:\n rake gitlab:sidekiq:drop_post_receive[group/project]" - end - project_path = Project.find_by_full_path(args.project).repository.path_to_repo - - Sidekiq.redis do |redis| - unless redis.exists(QUEUE) - abort "Queue #{QUEUE} is empty" - end - - temp_queue = "#{QUEUE}_#{Time.now.to_i}" - redis.rename(QUEUE, temp_queue) - - # At this point, then post_receive queue is empty. It may be receiving - # new jobs already. We will repopulate it with the old jobs, skipping the - # ones we want to drop. - dropped = 0 - while (job = redis.lpop(temp_queue)) - if repo_path(job) == project_path - dropped += 1 - else - redis.rpush(QUEUE, job) - end - end - # The temp_queue will delete itself after we have popped all elements - # from it - - puts "Dropped #{dropped} jobs containing #{project_path} from #{QUEUE}" - end - end - - def repo_path(job) - job_args = JSON.parse(job)['args'] - if job_args - job_args.first - else - nil - end - end - end -end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index e05be4a3405..8ac73bc8ff2 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -2,10 +2,10 @@ namespace :gitlab do namespace :storage do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do - legacy_projects_count = Project.with_legacy_storage.count + legacy_projects_count = Project.with_unmigrated_storage.count if legacy_projects_count == 0 - puts 'There are no projects using legacy storage. Nothing to do!' + puts 'There are no projects requiring storage migration. Nothing to do!' next end @@ -23,22 +23,42 @@ namespace :gitlab do desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' task legacy_projects: :environment do - projects_summary(Project.with_legacy_storage) + relation_summary('projects', Project.without_storage_feature(:repository)) end desc 'Gitlab | Storage | List existing projects using Legacy Storage' task list_legacy_projects: :environment do - projects_list(Project.with_legacy_storage) + projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository)) end desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' task hashed_projects: :environment do - projects_summary(Project.with_hashed_storage) + relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository)) end desc 'Gitlab | Storage | List existing projects using Hashed Storage' task list_hashed_projects: :environment do - projects_list(Project.with_hashed_storage) + projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository)) + end + + desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage' + task legacy_attachments: :environment do + relation_summary('attachments using Legacy Storage', legacy_attachments_relation) + end + + desc 'Gitlab | Storage | List existing project attachments using Legacy Storage' + task list_legacy_attachments: :environment do + attachments_list('attachments using Legacy Storage', legacy_attachments_relation) + end + + desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage' + task hashed_attachments: :environment do + relation_summary('attachments using Hashed Storage', hashed_attachments_relation) + end + + desc 'Gitlab | Storage | List existing project attachments using Hashed Storage' + task list_hashed_attachments: :environment do + attachments_list('attachments using Hashed Storage', hashed_attachments_relation) end def batch_size @@ -46,29 +66,43 @@ namespace :gitlab do end def project_id_batches(&block) - Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) yield ids.min, ids.max end end - def projects_summary(relation) - projects_count = relation.count - puts "* Found #{projects_count} projects".color(:green) + def legacy_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def hashed_attachments_relation + Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) + JOIN projects + ON (uploads.model_type='Project' AND uploads.model_id=projects.id) + SQL + end + + def relation_summary(relation_name, relation) + relation_count = relation.count + puts "* Found #{relation_count} #{relation_name}".color(:green) - projects_count + relation_count end - def projects_list(relation) - projects_count = projects_summary(relation) + def projects_list(relation_name, relation) + relation_count = relation_summary(relation_name, relation) projects = relation.with_route limit = ENV.fetch('LIMIT', 500).to_i - return unless projects_count > 0 + return unless relation_count > 0 - puts " ! Displaying first #{limit} projects..." if projects_count > limit + puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit counter = 0 projects.find_in_batches(batch_size: batch_size) do |batch| @@ -81,5 +115,26 @@ namespace :gitlab do end end end + + def attachments_list(relation_name, relation) + relation_count = relation_summary(relation_name, relation) + + limit = ENV.fetch('LIMIT', 500).to_i + + return unless relation_count > 0 + + puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit + + counter = 0 + relation.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |upload| + counter += 1 + + puts " - #{upload.path} (id: #{upload.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 7f86fd7b45e..aafbe52e5f8 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -7,14 +7,16 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @options = { token: token, verbose: true } + @options = { token: token } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first end def run! - @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one! + @repo = GithubRepos + .new(@options[:token], @current_user, @github_repo) + .choose_one! raise 'No repo found!' unless @repo @@ -28,7 +30,7 @@ class GithubImport private def show_warning! - puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" + puts "This will import GitHub #{@repo.full_name.bright} into GitLab #{@project_path.bright} as #{@current_user.name}" puts "Permission checks are ignored. Press any key to continue.".color(:red) STDIN.getch @@ -42,7 +44,9 @@ class GithubImport import_success = false timings = Benchmark.measure do - import_success = Github::Import.new(@project, @options).execute + import_success = Gitlab::GithubImport::SequentialImporter + .new(@project, token: @options[:token]) + .execute end if import_success @@ -63,16 +67,16 @@ class GithubImport @current_user, name: name, path: name, - description: @repo['description'], + description: @repo.description, namespace_id: namespace.id, visibility_level: visibility_level, - skip_wiki: @repo['has_wiki'] + skip_wiki: @repo.has_wiki ).execute project.update!( import_type: 'github', - import_source: @repo['full_name'], - import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@") + import_source: @repo.full_name, + import_url: @repo.clone_url.sub('://', "://#{@options[:token]}@") ) project @@ -91,13 +95,15 @@ class GithubImport end def visibility_level - @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility + @repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility end end class GithubRepos - def initialize(options, current_user, github_repo) - @options = options + def initialize(token, current_user, github_repo) + @client = Gitlab::GithubImport::Client.new(token) + @client.octokit.auto_paginate = true + @current_user = current_user @github_repo = github_repo end @@ -106,17 +112,17 @@ class GithubRepos return found_github_repo if @github_repo repos.each do |repo| - print "ID: #{repo['id'].to_s.bright}".color(:green) - print "\tName: #{repo['full_name']}\n".color(:green) + print "ID: #{repo.id.to_s.bright}".color(:green) + print "\tName: #{repo.full_name}\n".color(:green) end print 'ID? '.bright - repos.find { |repo| repo['id'] == repo_id } + repos.find { |repo| repo.id == repo_id } end def found_github_repo - repos.find { |repo| repo['full_name'] == @github_repo } + repos.find { |repo| repo.full_name == @github_repo } end def repo_id @@ -124,7 +130,7 @@ class GithubRepos end def repos - Github::Repositories.new(@options).fetch + @client.octokit.list_repositories end end |