diff options
Diffstat (limited to 'lib')
34 files changed, 476 insertions, 209 deletions
diff --git a/lib/api/files.rb b/lib/api/files.rb index 28f306e45f3..532a317c89e 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,8 +1,6 @@ module API # Projects API class Files < Grape::API - before { authenticate! } - helpers do def commit_params(attrs) { diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index c287ee34a68..4ca6646a6f1 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -2,7 +2,6 @@ require 'mime/types' module API class Repositories < Grape::API - before { authenticate! } before { authorize! :download_code, user_project } params do @@ -79,8 +78,6 @@ module API optional :format, type: String, desc: 'The archive format' end get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do - authorize! :download_code, user_project - begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue @@ -96,7 +93,6 @@ module API requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' end get ':id/repository/compare' do - authorize! :download_code, user_project compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) present compare, with: Entities::Compare end @@ -105,8 +101,6 @@ module API success Entities::Contributor end get ':id/repository/contributors' do - authorize! :download_code, user_project - begin present user_project.repository.contributors, with: Entities::Contributor diff --git a/lib/api/services.rb b/lib/api/services.rb index 59232c84c24..d11cdce4e18 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -378,7 +378,6 @@ module API desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' }, ], - 'mattermost-slash-commands' => [ { required: true, @@ -387,6 +386,14 @@ module API desc: 'The Mattermost token' } ], + 'slack-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Slack token' + } + ], 'pipelines-email' => [ { required: true, @@ -473,7 +480,7 @@ module API desc: 'The description of the tracker' } ], - 'slack-notification' => [ + 'slack' => [ { required: true, name: :webhook, @@ -493,7 +500,7 @@ module API desc: 'The channel name' } ], - 'mattermost-notification' => [ + 'mattermost' => [ { required: true, name: :webhook, diff --git a/lib/api/users.rb b/lib/api/users.rb index 0842c3874c5..4c22287b5c6 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -94,7 +94,7 @@ module API identity_attrs = params.slice(:provider, :extern_uid) confirm = params.delete(:confirm) - user = User.build_user(declared_params(include_missing: false)) + user = User.new(declared_params(include_missing: false)) user.skip_confirmation! unless confirm if identity_attrs.any? diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 2f19b59e725..d67d466bce8 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -10,7 +10,7 @@ module Banzai node.set_attribute('href', href) end - if href =~ /\Ahttp(s)?:\/\// && external_url?(href) + if href =~ %r{\A(https?:)?//[^/]} && external_url?(href) node.set_attribute('rel', 'nofollow noreferrer') node.set_attribute('target', '_blank') end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index ed87a2603e8..142bce82286 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -41,7 +41,7 @@ module Ci put ":id" do authenticate_runner! build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - forbidden!('Build has been erased!') if build.erased? + validate_build!(build) update_runner_info @@ -71,9 +71,7 @@ module Ci # PATCH /builds/:id/trace.txt patch ":id/trace.txt" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) - forbidden!('Build has been erased!') if build.erased? + authenticate_build!(build) error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] @@ -104,8 +102,7 @@ module Ci Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('build is not running') unless build.running? if params[:filesize] @@ -142,10 +139,8 @@ module Ci require_gitlab_workhorse! not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('Build is not running!') unless build.running? - forbidden!('Build has been erased!') if build.erased? artifacts_upload_path = ArtifactUploader.artifacts_upload_path artifacts = uploaded_file(:file, artifacts_upload_path) @@ -176,8 +171,7 @@ module Ci # GET /builds/:id/artifacts get ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) artifacts_file = build.artifacts_file unless artifacts_file.file_storage? @@ -202,8 +196,7 @@ module Ci # DELETE /builds/:id/artifacts delete ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) build.erase_artifacts! end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index e608f5f6cad..5ff25a3a9b2 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -13,8 +13,19 @@ module Ci forbidden! unless current_runner end - def authenticate_build_token!(build) - forbidden! unless build_token_valid?(build) + def authenticate_build!(build) + validate_build!(build) do + forbidden! unless build_token_valid?(build) + end + end + + def validate_build!(build) + not_found! unless build + + yield if block_given? + + forbidden!('Project has been deleted!') unless build.project + forbidden!('Build has been erased!') if build.erased? end def runner_registration_token_valid? @@ -49,7 +60,7 @@ module Ci end def build_not_found! - if headers['User-Agent'].match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) + if headers['User-Agent'].to_s.match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) no_content! else not_found! diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb index 25da8474e95..4fe53ce93a9 100644 --- a/lib/gitlab/chat_commands/base_command.rb +++ b/lib/gitlab/chat_commands/base_command.rb @@ -42,6 +42,10 @@ module Gitlab def find_by_iid(iid) collection.find_by(iid: iid) end + + def presenter + Gitlab::ChatCommands::Presenter.new + end end end end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index b0d3fdbc48a..145086755e4 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -22,8 +22,6 @@ module Gitlab end end - private - def match_command match = nil service = available_commands.find do |klass| @@ -33,6 +31,8 @@ module Gitlab [service, match] end + private + def help_messages available_commands.map(&:help_message) end @@ -48,15 +48,15 @@ module Gitlab end def help(messages) - Mattermost::Presenter.help(messages, params[:command]) + presenter.help(messages, params[:command]) end def access_denied - Mattermost::Presenter.access_denied + presenter.access_denied end def present(resource) - Mattermost::Presenter.present(resource) + presenter.present(resource) end end end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb index 0eed1fce0dc..6bb854dc080 100644 --- a/lib/gitlab/chat_commands/deploy.rb +++ b/lib/gitlab/chat_commands/deploy.rb @@ -4,7 +4,7 @@ module Gitlab include Gitlab::Routing.url_helpers def self.match(text) - /\Adeploy\s+(?<from>.*)\s+to+\s+(?<to>.*)\z/.match(text) + /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) end def self.help_message diff --git a/lib/mattermost/presenter.rb b/lib/gitlab/chat_commands/presenter.rb index 67eda983a74..caceaa25391 100644 --- a/lib/mattermost/presenter.rb +++ b/lib/gitlab/chat_commands/presenter.rb @@ -1,7 +1,7 @@ -module Mattermost - class Presenter - class << self - include Gitlab::Routing.url_helpers +module Gitlab + module ChatCommands + class Presenter + include Gitlab::Routing def authorize_chat_name(url) message = if url @@ -64,7 +64,7 @@ module Mattermost def single_resource(resource) return error(resource) if resource.errors.any? || !resource.persisted? - message = "### #{title(resource)}" + message = "#{title(resource)}:" message << "\n\n#{resource.description}" if resource.try(:description) in_channel_response(message) diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index c6bb8f9c8ed..9d142f1b82e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -45,7 +45,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 25e9d619697..79dd0cf7df2 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -22,7 +22,7 @@ module Gitlab def valid? environment_variables.all? do |(name, value)| - value.start_with?(project.repository.path_to_repo) + value.to_s.start_with?(project.repository.path_to_repo) end end @@ -35,7 +35,7 @@ module Gitlab end def environment_variables - @environment_variables ||= env.slice(*ALLOWED_VARIABLES) + @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact end end end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 6dbae64a9fe..95dba9a327b 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -15,6 +15,10 @@ module Gitlab end end + def url + raw_data.url || '' + end + private def gitlab_user_id(github_id) diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 85df6547a67..ba869faa92e 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,10 +4,12 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :access_token + attr_reader :access_token, :host, :api_version - def initialize(access_token) + def initialize(access_token, host: nil, api_version: 'v3') @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version if access_token ::Octokit.auto_paginate = false @@ -17,7 +19,7 @@ module Gitlab def api @api ||= ::Octokit::Client.new( access_token: access_token, - api_endpoint: github_options[:site], + api_endpoint: api_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { @@ -64,6 +66,14 @@ module Gitlab private + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + def config Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 281b65bdeba..ec1318ab33c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,7 +3,7 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :errors, :project, :repo, :repo_url + attr_reader :errors, :project, :repo, :repo_url def initialize(project) @project = project @@ -11,12 +11,27 @@ module Gitlab @repo_url = project.import_url @errors = [] @labels = {} + end + + def client + return @client if defined?(@client) + unless credentials + raise Projects::ImportService::Error, + "Unable to find project import data credentials for project ID: #{@project.id}" + end - if credentials - @client = Client.new(credentials[:user]) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + opts = {} + # Gitea plan to be GitHub compliant + if project.gitea_import? + uri = URI.parse(project.import_url) + host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') + opts = { + host: host, + api_version: 'v1' + } end + + @client = Client.new(credentials[:user], opts) end def execute @@ -35,7 +50,13 @@ module Gitlab import_comments(:issues) import_comments(:pull_requests) import_wiki - import_releases + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + handle_errors true @@ -44,7 +65,9 @@ module Gitlab private def credentials - @credentials ||= project.import_data.credentials if project.import_data + return @credentials if defined?(@credentials) + + @credentials = project.import_data ? project.import_data.credentials : nil end def handle_errors @@ -60,9 +83,10 @@ module Gitlab fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| begin - LabelFormatter.new(project, raw).create! + gh_label = LabelFormatter.new(project, raw) + gh_label.create! rescue => e - errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } end end end @@ -74,9 +98,10 @@ module Gitlab fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| begin - MilestoneFormatter.new(project, raw).create! + gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone.create! rescue => e - errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } end end end @@ -97,7 +122,7 @@ module Gitlab apply_labels(issuable, raw) rescue => e - errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } end end end @@ -106,18 +131,23 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| - pull_request = PullRequestFormatter.new(project, raw) - next unless pull_request.valid? + gh_pull_request = PullRequestFormatter.new(project, raw) + next unless gh_pull_request.valid? begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? + restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists? + restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists? + + merge_request = gh_pull_request.create! - pull_request.create! + # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage + if project.gitea_import? + apply_labels(merge_request, raw) + end rescue => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } ensure - clean_up_restored_branches(pull_request) + clean_up_restored_branches(gh_pull_request) end end end @@ -233,7 +263,7 @@ module Gitlab gh_release = ReleaseFormatter.new(project, raw) gh_release.create! if gh_release.valid? rescue => e - errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } end end end diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb new file mode 100644 index 00000000000..256f360efc7 --- /dev/null +++ b/lib/gitlab/github_import/issuable_formatter.rb @@ -0,0 +1,60 @@ +module Gitlab + module GithubImport + class IssuableFormatter < BaseFormatter + def project_association + raise NotImplementedError + end + + def number + raw_data.number + end + + def find_condition + { iid: number } + end + + private + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gitlab_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gitlab_author_id || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + if gitlab_author_id + body + else + formatter.author_line(author) + body + end + end + + def milestone + if raw_data.milestone.present? + milestone = MilestoneFormatter.new(project, raw_data.milestone) + project.milestones.find_by(milestone.find_condition) + end + end + end + end +end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 887690bcc7c..6f5ac4dac0d 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class IssueFormatter < BaseFormatter + class IssueFormatter < IssuableFormatter def attributes { iid: number, @@ -24,59 +24,9 @@ module Gitlab :issues end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def pull_request? raw_data.pull_request.present? end - - private - - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body - else - formatter.author_line(author) + body - end - end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - raw_data.state == 'closed' ? 'closed' : 'opened' - end end end end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 401dd962521..dd782eff059 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,7 +3,7 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: raw_data.number, + iid: number, project: project, title: raw_data.title, description: raw_data.description, @@ -19,7 +19,15 @@ module Gitlab end def find_condition - { iid: raw_data.number } + { iid: number } + end + + def number + if project.gitea_import? + raw_data.id + else + raw_data.number + end end private diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index a2410068845..3f635be22ba 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,14 +1,15 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :name, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type - def initialize(repo, name, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data, type: 'github') @repo = repo @name = name @namespace = namespace @current_user = current_user @session_data = session_data + @type = type end def execute @@ -19,7 +20,7 @@ module Gitlab description: repo.description, namespace_id: namespace.id, visibility_level: visibility_level, - import_type: "github", + import_type: type, import_source: repo.full_name, import_url: import_url, skip_wiki: skip_wiki @@ -29,7 +30,7 @@ module Gitlab private def import_url - repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") end def visibility_level diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b9a227fb11a..4ea0200e89b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class PullRequestFormatter < BaseFormatter + class PullRequestFormatter < IssuableFormatter delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true @@ -28,14 +28,6 @@ module Gitlab :merge_requests end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def valid? source_branch.valid? && target_branch.valid? end @@ -60,57 +52,15 @@ module Gitlab end end - def url - raw_data.url - end - private - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body + def state + if raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' else - formatter.author_line(author) + body + super end end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - elsif raw_data.state == 'closed' - 'closed' - else - 'opened' - end - end end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 65b229ca8ff..7a649f28340 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -22,7 +22,7 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels project_label group_label].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze def self.create(*args) new(*args).create @@ -189,7 +189,7 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - attribute_hash = attribute_hash_for(['events', 'priorities']) + attribute_hash = attribute_hash_for(['events']) existing_object.assign_attributes(attribute_hash) if attribute_hash.any? @@ -210,9 +210,8 @@ module Gitlab def existing_object @existing_object ||= begin - finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] - finder_hash = parsed_relation_hash.slice(*finder_attributes) - existing_object = relation_class.find_or_create_by(finder_hash) + existing_object = find_or_create_object! + # Done in two steps, as MySQL behaves differently than PostgreSQL using # the +find_or_create_by+ method and does not return the ID the second time. existing_object.update!(parsed_relation_hash) @@ -224,6 +223,25 @@ module Gitlab @relation_name == :services && parsed_relation_hash['type'] && !Object.const_defined?(parsed_relation_hash['type']) end + + def find_or_create_object! + finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_hash = parsed_relation_hash.slice(*finder_attributes) + + if label? + label = relation_class.find_or_initialize_by(finder_hash) + parsed_relation_hash.delete('priorities') if label.persisted? + + label.save! + label + else + relation_class.find_or_create_by(finder_hash) + end + end + + def label? + @relation_name.to_s.include?('label') + end end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 94261b7eeed..45958710c13 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,21 +7,38 @@ module Gitlab module ImportSources extend CurrentSettings + ImportSource = Struct.new(:name, :title, :importer) + + ImportTable = [ + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer), + ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repo by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ].freeze + class << self + def options + @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + end + def values - options.values + @values ||= ImportTable.map(&:name) end - def options - { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', - 'GitLab export' => 'gitlab_project' - } + def importer_names + @importer_names ||= ImportTable.select(&:importer).map(&:name) + end + + def importer(name) + ImportTable.find { |import_source| import_source.name == name }.importer + end + + def title(name) + options.key(name) end end end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb new file mode 100644 index 00000000000..288771c1c12 --- /dev/null +++ b/lib/gitlab/kubernetes.rb @@ -0,0 +1,80 @@ +module Gitlab + # Helper methods to do with Kubernetes network services & resources + module Kubernetes + # This is the comand that is run to start a terminal session. Kubernetes + # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` + EXEC_COMMAND = URI.encode_www_form( + ['sh', '-c', 'bash || sh'].map { |value| ['command', value] } + ) + + # Filters an array of pods (as returned by the kubernetes API) by their labels + def filter_pods(pods, labels = {}) + pods.select do |pod| + metadata = pod.fetch("metadata", {}) + pod_labels = metadata.fetch("labels", nil) + next unless pod_labels + + labels.all? { |k, v| pod_labels[k.to_s] == v } + end + end + + # Converts a pod (as returned by the kubernetes API) into a terminal + def terminals_for_pod(api_url, namespace, pod) + metadata = pod.fetch("metadata", {}) + status = pod.fetch("status", {}) + spec = pod.fetch("spec", {}) + + containers = spec["containers"] + pod_name = metadata["name"] + phase = status["phase"] + + return unless containers.present? && pod_name.present? && phase == "Running" + + created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil + + containers.map do |container| + { + selectors: { pod: pod_name, container: container["name"] }, + url: container_exec_url(api_url, namespace, pod_name, container["name"]), + subprotocols: ['channel.k8s.io'], + headers: Hash.new { |h, k| h[k] = [] }, + created_at: created_at, + } + end + end + + def add_terminal_auth(terminal, token, ca_pem = nil) + terminal[:headers]['Authorization'] << "Bearer #{token}" + terminal[:ca_pem] = ca_pem if ca_pem.present? + terminal + end + + def container_exec_url(api_url, namespace, pod_name, container_name) + url = URI.parse(api_url) + url.path = [ + url.path.sub(%r{/+\z}, ''), + 'api', 'v1', + 'namespaces', ERB::Util.url_encode(namespace), + 'pods', ERB::Util.url_encode(pod_name), + 'exec' + ].join('/') + + url.query = { + container: container_name, + tty: true, + stdin: true, + stdout: true, + stderr: true, + }.to_query + '&' + EXEC_COMMAND + + case url.scheme + when 'http' + url.scheme = 'ws' + when 'https' + url.scheme = 'wss' + end + + url.to_s + end + end +end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 01c96a6fe96..91fb0bb317a 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -70,8 +70,8 @@ module Gitlab def tag_endpoint(trans, env) endpoint = env[ENDPOINT_KEY] - path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path] - trans.action = "Grape##{endpoint.route.route_method} #{path}" + path = endpoint_paths_cache[endpoint.route.request_method][endpoint.route.path] + trans.action = "Grape##{endpoint.route.request_method} #{path}" end private diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 65713e73a59..dd99f9bb7d7 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -42,7 +42,7 @@ module Gitlab key, value = parsed_field.first if value.nil? - value = File.open(tmp_path) + value = open_file(tmp_path) @open_files << value else value = decorate_params_value(value, @request.params[key], tmp_path) @@ -68,7 +68,7 @@ module Gitlab case path_value when nil - value_hash[path_key] = File.open(tmp_path) + value_hash[path_key] = open_file(tmp_path) @open_files << value_hash[path_key] value_hash when Hash @@ -78,6 +78,10 @@ module Gitlab raise "unexpected path value: #{path_value.inspect}" end end + + def open_file(path) + ::UploadedFile.new(path, File.basename(path), 'application/octet-stream') + end end def initialize(app) diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 8d1a1ed54c9..d19b0a52043 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -13,8 +13,9 @@ module Gitlab def categories { - "General" => '', - "Pages" => 'Pages' + 'General' => '', + 'Pages' => 'Pages', + 'Autodeploy' => 'autodeploy' } end @@ -25,6 +26,11 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) end + + def dropdown_names(context) + categories = context == 'autodeploy' ? ['Autodeploy'] : ['General', 'Pages'] + super().slice(*categories) + end end end end diff --git a/lib/gitlab/update_path_error.rb b/lib/gitlab/update_path_error.rb new file mode 100644 index 00000000000..ce14cc887d0 --- /dev/null +++ b/lib/gitlab/update_path_error.rb @@ -0,0 +1,3 @@ +module Gitlab + class UpdatePathError < StandardError; end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index aeb1a26e1ba..d28bb583fe7 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -95,6 +95,19 @@ module Gitlab ] end + def terminal_websocket(terminal) + details = { + 'Terminal' => { + 'Subprotocols' => terminal[:subprotocols], + 'Url' => terminal[:url], + 'Header' => terminal[:headers] + } + } + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + + details + end + def version path = Rails.root.join(VERSION_FILE) path.readable? ? path.read.chomp : 'unknown' diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb new file mode 100644 index 00000000000..ec2903b7ec6 --- /dev/null +++ b/lib/mattermost/client.rb @@ -0,0 +1,41 @@ +module Mattermost + class ClientError < Mattermost::Error; end + + class Client + attr_reader :user + + def initialize(user) + @user = user + end + + private + + def with_session(&blk) + Mattermost::Session.new(user).with_session(&blk) + end + + def json_get(path, options = {}) + with_session do |session| + json_response session.get(path, options) + end + end + + def json_post(path, options = {}) + with_session do |session| + json_response session.post(path, options) + end + end + + def json_response(response) + json_response = JSON.parse(response.body) + + unless response.success? + raise Mattermost::ClientError.new(json_response['message'] || 'Undefined error') + end + + json_response + rescue JSON::JSONError + raise Mattermost::ClientError.new('Cannot parse response') + end + end +end diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb new file mode 100644 index 00000000000..d1e4bb0eccf --- /dev/null +++ b/lib/mattermost/command.rb @@ -0,0 +1,10 @@ +module Mattermost + class Command < Client + def create(params) + response = json_post("/api/v3/teams/#{params[:team_id]}/commands/create", + body: params.to_json) + + response['token'] + end + end +end diff --git a/lib/mattermost/error.rb b/lib/mattermost/error.rb new file mode 100644 index 00000000000..014df175be0 --- /dev/null +++ b/lib/mattermost/error.rb @@ -0,0 +1,3 @@ +module Mattermost + class Error < StandardError; end +end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index fb8d7d97f8a..377cb7b1021 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -1,5 +1,12 @@ module Mattermost - class NoSessionError < StandardError; end + class NoSessionError < Mattermost::Error + def message + 'No session could be set up, is Mattermost configured with Single Sign On?' + end + end + + class ConnectionError < Mattermost::Error; end + # This class' prime objective is to obtain a session token on a Mattermost # instance with SSO configured where this GitLab instance is the provider. # @@ -17,6 +24,8 @@ module Mattermost include Doorkeeper::Helpers::Controller include HTTParty + LEASE_TIMEOUT = 60 + base_uri Settings.mattermost.host attr_accessor :current_resource_owner, :token @@ -26,12 +35,16 @@ module Mattermost end def with_session - raise NoSessionError unless create - - begin - yield self - ensure - destroy + with_lease do + raise Mattermost::NoSessionError unless create + + begin + yield self + rescue Errno::ECONNREFUSED + raise Mattermost::NoSessionError + ensure + destroy + end end end @@ -58,11 +71,15 @@ module Mattermost end def get(path, options = {}) - self.class.get(path, options.merge(headers: @headers)) + handle_exceptions do + self.class.get(path, options.merge(headers: @headers)) + end end def post(path, options = {}) - self.class.post(path, options.merge(headers: @headers)) + handle_exceptions do + self.class.post(path, options.merge(headers: @headers)) + end end private @@ -111,5 +128,33 @@ module Mattermost response.headers['token'] end end + + def with_lease + lease_uuid = lease_try_obtain + raise NoSessionError unless lease_uuid + + begin + yield + ensure + Gitlab::ExclusiveLease.cancel(lease_key, lease_uuid) + end + end + + def lease_key + "mattermost:session" + end + + def lease_try_obtain + lease = ::Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT) + lease.try_obtain + end + + def handle_exceptions + yield + rescue HTTParty::Error => e + raise Mattermost::ConnectionError.new(e.message) + rescue Errno::ECONNREFUSED + raise Mattermost::ConnectionError.new(e.message) + end end end diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb new file mode 100644 index 00000000000..784eca6ab5a --- /dev/null +++ b/lib/mattermost/team.rb @@ -0,0 +1,7 @@ +module Mattermost + class Team < Client + def all + json_get('/api/v3/teams/all') + end + end +end |