From 3cc334eae5abf65b0ecb4159e4e9712dab81c33c Mon Sep 17 00:00:00 2001 From: Arsenev Vladislav Date: Sun, 18 Dec 2016 21:56:13 +0000 Subject: remove build_user from model User --- lib/api/users.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index 0842c3874c5..4c22287b5c6 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -94,7 +94,7 @@ module API identity_attrs = params.slice(:provider, :extern_uid) confirm = params.delete(:confirm) - user = User.build_user(declared_params(include_missing: false)) + user = User.new(declared_params(include_missing: false)) user.skip_confirmation! unless confirm if identity_attrs.any? -- cgit v1.2.1 From 5d4531db2555d3051fc47e9268728a670ece95f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Mon, 17 Oct 2016 17:58:57 +0200 Subject: Gogs Importer --- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/github_import/client.rb | 14 ++++-- lib/gitlab/github_import/importer.rb | 16 +++---- lib/gitlab/github_import/issue_formatter.rb | 2 +- lib/gitlab/github_import/milestone_formatter.rb | 8 +++- lib/gitlab/github_import/project_creator.rb | 9 ++-- lib/gitlab/github_import/pull_request_formatter.rb | 2 +- lib/gitlab/gogs_import/importer.rb | 54 ++++++++++++++++++++++ lib/gitlab/gogs_import/milestone_formatter.rb | 9 ++++ lib/gitlab/import_sources.rb | 1 + 10 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 lib/gitlab/gogs_import/importer.rb create mode 100644 lib/gitlab/gogs_import/milestone_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index c6bb8f9c8ed..eb3d9f29451 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -45,7 +45,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + import_sources: %w[gogs github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 85df6547a67..a96e0bc63dd 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,24 +4,30 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :access_token + attr_reader :access_token, :host, :api_version - def initialize(access_token) + def initialize(access_token, host: nil, api_version: 'v3') @access_token = access_token + @host = host + @api_version = api_version if access_token ::Octokit.auto_paginate = false end end + def api_endpoint + host.present? && api_version.present? ? "#{host}/api/#{api_version}" : github_options[:site] + end + def api @api ||= ::Octokit::Client.new( access_token: access_token, - api_endpoint: github_options[:site], + api_endpoint: api_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { - ssl: { verify: config ? config['verify_ssl'] : true } + ssl: { verify: config ? config['verify_ssl'] : false } } ) end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 281b65bdeba..c32e78cae03 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -60,7 +60,7 @@ module Gitlab fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| begin - LabelFormatter.new(project, raw).create! + GithubImport::LabelFormatter.new(project, raw).create! rescue => e errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } end @@ -74,7 +74,7 @@ module Gitlab fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| begin - MilestoneFormatter.new(project, raw).create! + GithubImport::MilestoneFormatter.new(project, raw).create! rescue => e errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } end @@ -85,7 +85,7 @@ module Gitlab def import_issues fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| - gh_issue = IssueFormatter.new(project, raw) + gh_issue = GithubImport::IssueFormatter.new(project, raw) begin issuable = @@ -106,7 +106,7 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| - pull_request = PullRequestFormatter.new(project, raw) + pull_request = GithubImport::PullRequestFormatter.new(project, raw) next unless pull_request.valid? begin @@ -179,7 +179,7 @@ module Gitlab ActiveRecord::Base.no_touching do comments.each do |raw| begin - comment = CommentFormatter.new(project, raw) + comment = GithubImport::CommentFormatter.new(project, raw) # GH does not return info about comment's parent, so we guess it by checking its URL! *_, parent, iid = URI(raw.html_url).path.split('/') issuable_class = parent == 'issues' ? Issue : MergeRequest @@ -198,7 +198,7 @@ module Gitlab last_note_attrs = nil cut_off_index = comments.find_index do |raw| - comment = CommentFormatter.new(project, raw) + comment = GithubImport::CommentFormatter.new(project, raw) comment_attrs = comment.attributes last_note_attrs ||= last_note.slice(*comment_attrs.keys) @@ -214,7 +214,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? - wiki = WikiFormatter.new(project) + wiki = GithubImport::WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) end rescue Gitlab::Shell::Error => e @@ -230,7 +230,7 @@ module Gitlab fetch_resources(:releases, repo, per_page: 100) do |releases| releases.each do |raw| begin - gh_release = ReleaseFormatter.new(project, raw) + gh_release = GithubImport::ReleaseFormatter.new(project, raw) gh_release.create! if gh_release.valid? rescue => e errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 887690bcc7c..21a3dee203b 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -70,7 +70,7 @@ module Gitlab def milestone if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) + project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr)) end end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 401dd962521..678d56b830b 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,7 +3,7 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: raw_data.number, + iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr), project: project, title: raw_data.title, description: raw_data.description, @@ -19,7 +19,11 @@ module Gitlab end def find_condition - { iid: raw_data.number } + { iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr) } + end + + def self.iid_attr + :number end private diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index a2410068845..3f635be22ba 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,14 +1,15 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :name, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type - def initialize(repo, name, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data, type: 'github') @repo = repo @name = name @namespace = namespace @current_user = current_user @session_data = session_data + @type = type end def execute @@ -19,7 +20,7 @@ module Gitlab description: repo.description, namespace_id: namespace.id, visibility_level: visibility_level, - import_type: "github", + import_type: type, import_source: repo.full_name, import_url: import_url, skip_wiki: skip_wiki @@ -29,7 +30,7 @@ module Gitlab private def import_url - repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") end def visibility_level diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b9a227fb11a..ea8768fded7 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -98,7 +98,7 @@ module Gitlab def milestone if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) + project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr)) end end diff --git a/lib/gitlab/gogs_import/importer.rb b/lib/gitlab/gogs_import/importer.rb new file mode 100644 index 00000000000..604e31d35a3 --- /dev/null +++ b/lib/gitlab/gogs_import/importer.rb @@ -0,0 +1,54 @@ +require 'uri' + +module Gitlab + module GogsImport + class Importer < Gitlab::GithubImport::Importer + include Gitlab::ShellAdapter + + attr_reader :client, :errors, :project, :repo, :repo_url + + def initialize(project) + @project = project + @repo = project.import_source + @repo_url = project.import_url + @errors = [] + @labels = {} + + if credentials + uri = URI.parse(project.import_url) + host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(/[\w-]+\/[\w-]+\.git\z/, '') + @client = GithubImport::Client.new(credentials[:user], host: host, api_version: 'v1') + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end + end + + def execute + import_labels + import_milestones + import_pull_requests + import_issues + import_comments(:issues) + import_comments(:pull_requests) + import_wiki + # NOTE: this is commented out since Gogs doesn't have release-API yet + # import_releases + handle_errors + + true + end + + def import_milestones + fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| + milestones.each do |raw| + begin + GogsImport::MilestoneFormatter.new(project, raw).create! + rescue => e + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end + end + end + end +end diff --git a/lib/gitlab/gogs_import/milestone_formatter.rb b/lib/gitlab/gogs_import/milestone_formatter.rb new file mode 100644 index 00000000000..990e792929a --- /dev/null +++ b/lib/gitlab/gogs_import/milestone_formatter.rb @@ -0,0 +1,9 @@ +module Gitlab + module GogsImport + class MilestoneFormatter < GithubImport::MilestoneFormatter + def self.iid_attr + :id + end + end + end +end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 94261b7eeed..9564c4cc134 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -14,6 +14,7 @@ module Gitlab def options { + 'Gogs' => 'gogs', 'GitHub' => 'github', 'Bitbucket' => 'bitbucket', 'GitLab.com' => 'gitlab', -- cgit v1.2.1 From 103114e3d73819f76bed9d8ad1bbdb8964875579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 15 Dec 2016 17:31:14 +0100 Subject: Rename Gogs to Gitea, DRY the controller and improve views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/gogs_import/importer.rb | 54 --------------------------- lib/gitlab/gogs_import/milestone_formatter.rb | 9 ----- lib/gitlab/import_sources.rb | 4 +- 4 files changed, 3 insertions(+), 66 deletions(-) delete mode 100644 lib/gitlab/gogs_import/importer.rb delete mode 100644 lib/gitlab/gogs_import/milestone_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index eb3d9f29451..9d142f1b82e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -45,7 +45,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[gogs github bitbucket gitlab google_code fogbugz git gitlab_project], + import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/gogs_import/importer.rb b/lib/gitlab/gogs_import/importer.rb deleted file mode 100644 index 604e31d35a3..00000000000 --- a/lib/gitlab/gogs_import/importer.rb +++ /dev/null @@ -1,54 +0,0 @@ -require 'uri' - -module Gitlab - module GogsImport - class Importer < Gitlab::GithubImport::Importer - include Gitlab::ShellAdapter - - attr_reader :client, :errors, :project, :repo, :repo_url - - def initialize(project) - @project = project - @repo = project.import_source - @repo_url = project.import_url - @errors = [] - @labels = {} - - if credentials - uri = URI.parse(project.import_url) - host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(/[\w-]+\/[\w-]+\.git\z/, '') - @client = GithubImport::Client.new(credentials[:user], host: host, api_version: 'v1') - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" - end - end - - def execute - import_labels - import_milestones - import_pull_requests - import_issues - import_comments(:issues) - import_comments(:pull_requests) - import_wiki - # NOTE: this is commented out since Gogs doesn't have release-API yet - # import_releases - handle_errors - - true - end - - def import_milestones - fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| - milestones.each do |raw| - begin - GogsImport::MilestoneFormatter.new(project, raw).create! - rescue => e - errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } - end - end - end - end - end - end -end diff --git a/lib/gitlab/gogs_import/milestone_formatter.rb b/lib/gitlab/gogs_import/milestone_formatter.rb deleted file mode 100644 index 990e792929a..00000000000 --- a/lib/gitlab/gogs_import/milestone_formatter.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - module GogsImport - class MilestoneFormatter < GithubImport::MilestoneFormatter - def self.iid_attr - :id - end - end - end -end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 9564c4cc134..34587582bd1 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -14,14 +14,14 @@ module Gitlab def options { - 'Gogs' => 'gogs', 'GitHub' => 'github', 'Bitbucket' => 'bitbucket', 'GitLab.com' => 'gitlab', 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', 'Repo by URL' => 'git', - 'GitLab export' => 'gitlab_project' + 'GitLab export' => 'gitlab_project', + 'Gitea' => 'gitea' } end end -- cgit v1.2.1 From 99ddd1dcbed35b642d7bd8a52cc6e5e5453b9f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 15 Dec 2016 17:36:53 +0100 Subject: Modify GithubImport to support Gitea MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reason is that Gitea plan to be GitHub-compatible so it makes sense to just modify GitHubImport a bit for now, and hopefully we can change it to GitHubishImport once Gitea is 100%-compatible. Signed-off-by: Rémy Coutable --- lib/gitlab/github_import/base_formatter.rb | 4 ++ lib/gitlab/github_import/client.rb | 16 +++-- lib/gitlab/github_import/importer.rb | 80 +++++++++++++++------- lib/gitlab/github_import/issuable_formatter.rb | 60 ++++++++++++++++ lib/gitlab/github_import/issue_formatter.rb | 52 +------------- lib/gitlab/github_import/milestone_formatter.rb | 12 ++-- lib/gitlab/github_import/pull_request_formatter.rb | 60 ++-------------- 7 files changed, 143 insertions(+), 141 deletions(-) create mode 100644 lib/gitlab/github_import/issuable_formatter.rb (limited to 'lib') diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 6dbae64a9fe..95dba9a327b 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -15,6 +15,10 @@ module Gitlab end end + def url + raw_data.url || '' + end + private def gitlab_user_id(github_id) diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index a96e0bc63dd..ba869faa92e 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -8,7 +8,7 @@ module Gitlab def initialize(access_token, host: nil, api_version: 'v3') @access_token = access_token - @host = host + @host = host.to_s.sub(%r{/+\z}, '') @api_version = api_version if access_token @@ -16,10 +16,6 @@ module Gitlab end end - def api_endpoint - host.present? && api_version.present? ? "#{host}/api/#{api_version}" : github_options[:site] - end - def api @api ||= ::Octokit::Client.new( access_token: access_token, @@ -27,7 +23,7 @@ module Gitlab # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { - ssl: { verify: config ? config['verify_ssl'] : false } + ssl: { verify: config ? config['verify_ssl'] : true } } ) end @@ -70,6 +66,14 @@ module Gitlab private + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + def config Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index c32e78cae03..c53cd1a928d 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,7 +3,7 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :errors, :project, :repo, :repo_url + attr_reader :errors, :project, :repo, :repo_url def initialize(project) @project = project @@ -11,12 +11,27 @@ module Gitlab @repo_url = project.import_url @errors = [] @labels = {} + end + + def client + return @client if defined?(@client) + unless credentials + raise Projects::ImportService::Error, + "Unable to find project import data credentials for project ID: #{@project.id}" + end - if credentials - @client = Client.new(credentials[:user]) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + opts = {} + # Gitea plan to be GitHub compliant + if project.import_type == 'gitea' + uri = URI.parse(project.import_url) + host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') + opts = { + host: host, + api_version: 'v1' + } end + + @client = Client.new(credentials[:user], opts) end def execute @@ -35,7 +50,13 @@ module Gitlab import_comments(:issues) import_comments(:pull_requests) import_wiki - import_releases + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.import_type == 'gitea' + import_releases + end + handle_errors true @@ -44,7 +65,9 @@ module Gitlab private def credentials - @credentials ||= project.import_data.credentials if project.import_data + return @credentials if defined?(@credentials) + + @credentials = project.import_data ? project.import_data.credentials : nil end def handle_errors @@ -60,9 +83,10 @@ module Gitlab fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| begin - GithubImport::LabelFormatter.new(project, raw).create! + gh_label = LabelFormatter.new(project, raw) + gh_label.create! rescue => e - errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } end end end @@ -74,9 +98,10 @@ module Gitlab fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| begin - GithubImport::MilestoneFormatter.new(project, raw).create! + gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone.create! rescue => e - errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } end end end @@ -85,7 +110,7 @@ module Gitlab def import_issues fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| - gh_issue = GithubImport::IssueFormatter.new(project, raw) + gh_issue = IssueFormatter.new(project, raw) begin issuable = @@ -97,7 +122,7 @@ module Gitlab apply_labels(issuable, raw) rescue => e - errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } end end end @@ -106,18 +131,23 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| - pull_request = GithubImport::PullRequestFormatter.new(project, raw) - next unless pull_request.valid? + gh_pull_request = PullRequestFormatter.new(project, raw) + next unless gh_pull_request.valid? begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? + restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists? + restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists? + + merge_request = gh_pull_request.create! - pull_request.create! + # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage + if project.import_type == 'gitea' + apply_labels(merge_request, raw) + end rescue => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } ensure - clean_up_restored_branches(pull_request) + clean_up_restored_branches(gh_pull_request) end end end @@ -179,7 +209,7 @@ module Gitlab ActiveRecord::Base.no_touching do comments.each do |raw| begin - comment = GithubImport::CommentFormatter.new(project, raw) + comment = CommentFormatter.new(project, raw) # GH does not return info about comment's parent, so we guess it by checking its URL! *_, parent, iid = URI(raw.html_url).path.split('/') issuable_class = parent == 'issues' ? Issue : MergeRequest @@ -198,7 +228,7 @@ module Gitlab last_note_attrs = nil cut_off_index = comments.find_index do |raw| - comment = GithubImport::CommentFormatter.new(project, raw) + comment = CommentFormatter.new(project, raw) comment_attrs = comment.attributes last_note_attrs ||= last_note.slice(*comment_attrs.keys) @@ -214,7 +244,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? - wiki = GithubImport::WikiFormatter.new(project) + wiki = WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) end rescue Gitlab::Shell::Error => e @@ -230,10 +260,10 @@ module Gitlab fetch_resources(:releases, repo, per_page: 100) do |releases| releases.each do |raw| begin - gh_release = GithubImport::ReleaseFormatter.new(project, raw) + gh_release = ReleaseFormatter.new(project, raw) gh_release.create! if gh_release.valid? rescue => e - errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } end end end diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb new file mode 100644 index 00000000000..256f360efc7 --- /dev/null +++ b/lib/gitlab/github_import/issuable_formatter.rb @@ -0,0 +1,60 @@ +module Gitlab + module GithubImport + class IssuableFormatter < BaseFormatter + def project_association + raise NotImplementedError + end + + def number + raw_data.number + end + + def find_condition + { iid: number } + end + + private + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gitlab_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gitlab_author_id || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + if gitlab_author_id + body + else + formatter.author_line(author) + body + end + end + + def milestone + if raw_data.milestone.present? + milestone = MilestoneFormatter.new(project, raw_data.milestone) + project.milestones.find_by(milestone.find_condition) + end + end + end + end +end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 21a3dee203b..6f5ac4dac0d 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class IssueFormatter < BaseFormatter + class IssueFormatter < IssuableFormatter def attributes { iid: number, @@ -24,59 +24,9 @@ module Gitlab :issues end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def pull_request? raw_data.pull_request.present? end - - private - - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body - else - formatter.author_line(author) + body - end - end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr)) - end - end - - def state - raw_data.state == 'closed' ? 'closed' : 'opened' - end end end end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 678d56b830b..fe172b854d5 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,7 +3,7 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr), + iid: number, project: project, title: raw_data.title, description: raw_data.description, @@ -19,11 +19,15 @@ module Gitlab end def find_condition - { iid: raw_data.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr) } + { iid: number } end - def self.iid_attr - :number + def number + if project.import_type == 'gitea' + raw_data.id + else + raw_data.number + end end private diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index ea8768fded7..4ea0200e89b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class PullRequestFormatter < BaseFormatter + class PullRequestFormatter < IssuableFormatter delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true @@ -28,14 +28,6 @@ module Gitlab :merge_requests end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def valid? source_branch.valid? && target_branch.valid? end @@ -60,57 +52,15 @@ module Gitlab end end - def url - raw_data.url - end - private - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body + def state + if raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' else - formatter.author_line(author) + body + super end end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.public_send("Gitlab::#{project.import_type.camelize}Import::MilestoneFormatter".constantize.iid_attr)) - end - end - - def state - @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - elsif raw_data.state == 'closed' - 'closed' - else - 'opened' - end - end end end end -- cgit v1.2.1 From 8fc63d1f648fa38eac9e5422dd42667d8e7f1b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 16 Dec 2016 09:15:30 +0100 Subject: Improve Gitlab::ImportSources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/import_sources.rb | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 34587582bd1..45958710c13 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,22 +7,38 @@ module Gitlab module ImportSources extend CurrentSettings + ImportSource = Struct.new(:name, :title, :importer) + + ImportTable = [ + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer), + ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repo by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ].freeze + class << self + def options + @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + end + def values - options.values + @values ||= ImportTable.map(&:name) end - def options - { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', - 'GitLab export' => 'gitlab_project', - 'Gitea' => 'gitea' - } + def importer_names + @importer_names ||= ImportTable.select(&:importer).map(&:name) + end + + def importer(name) + ImportTable.find { |import_source| import_source.name == name }.importer + end + + def title(name) + options.key(name) end end end -- cgit v1.2.1 From ab06313c36fc5856b2472d3dfcb966a8c6341d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 16 Dec 2016 17:44:22 +0100 Subject: Add Project#gitea_import? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/github_import/importer.rb | 6 +++--- lib/gitlab/github_import/milestone_formatter.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index c53cd1a928d..ec1318ab33c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -22,7 +22,7 @@ module Gitlab opts = {} # Gitea plan to be GitHub compliant - if project.import_type == 'gitea' + if project.gitea_import? uri = URI.parse(project.import_url) host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') opts = { @@ -53,7 +53,7 @@ module Gitlab # Gitea doesn't have a Release API yet # See https://github.com/go-gitea/gitea/issues/330 - unless project.import_type == 'gitea' + unless project.gitea_import? import_releases end @@ -141,7 +141,7 @@ module Gitlab merge_request = gh_pull_request.create! # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage - if project.import_type == 'gitea' + if project.gitea_import? apply_labels(merge_request, raw) end rescue => e diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index fe172b854d5..dd782eff059 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -23,7 +23,7 @@ module Gitlab end def number - if project.import_type == 'gitea' + if project.gitea_import? raw_data.id else raw_data.number -- cgit v1.2.1 From 2e6c1720ead0f2843abb0d03f0c01b92fa063980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 16 Dec 2016 18:21:58 +0100 Subject: Allow Repositories API GET endpoints to be requested anonymously MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/api/repositories.rb | 6 ------ 1 file changed, 6 deletions(-) (limited to 'lib') diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index c287ee34a68..4ca6646a6f1 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -2,7 +2,6 @@ require 'mime/types' module API class Repositories < Grape::API - before { authenticate! } before { authorize! :download_code, user_project } params do @@ -79,8 +78,6 @@ module API optional :format, type: String, desc: 'The archive format' end get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do - authorize! :download_code, user_project - begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue @@ -96,7 +93,6 @@ module API requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' end get ':id/repository/compare' do - authorize! :download_code, user_project compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) present compare, with: Entities::Compare end @@ -105,8 +101,6 @@ module API success Entities::Contributor end get ':id/repository/contributors' do - authorize! :download_code, user_project - begin present user_project.repository.contributors, with: Entities::Contributor -- cgit v1.2.1 From c3d972f4e861059312c2708dacb57999416fcc70 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 22 Nov 2016 19:55:56 +0000 Subject: Add terminals to the Kubernetes deployment service --- lib/gitlab/kubernetes.rb | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ lib/gitlab/workhorse.rb | 13 ++++++++ 2 files changed, 93 insertions(+) create mode 100644 lib/gitlab/kubernetes.rb (limited to 'lib') diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb new file mode 100644 index 00000000000..288771c1c12 --- /dev/null +++ b/lib/gitlab/kubernetes.rb @@ -0,0 +1,80 @@ +module Gitlab + # Helper methods to do with Kubernetes network services & resources + module Kubernetes + # This is the comand that is run to start a terminal session. Kubernetes + # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` + EXEC_COMMAND = URI.encode_www_form( + ['sh', '-c', 'bash || sh'].map { |value| ['command', value] } + ) + + # Filters an array of pods (as returned by the kubernetes API) by their labels + def filter_pods(pods, labels = {}) + pods.select do |pod| + metadata = pod.fetch("metadata", {}) + pod_labels = metadata.fetch("labels", nil) + next unless pod_labels + + labels.all? { |k, v| pod_labels[k.to_s] == v } + end + end + + # Converts a pod (as returned by the kubernetes API) into a terminal + def terminals_for_pod(api_url, namespace, pod) + metadata = pod.fetch("metadata", {}) + status = pod.fetch("status", {}) + spec = pod.fetch("spec", {}) + + containers = spec["containers"] + pod_name = metadata["name"] + phase = status["phase"] + + return unless containers.present? && pod_name.present? && phase == "Running" + + created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil + + containers.map do |container| + { + selectors: { pod: pod_name, container: container["name"] }, + url: container_exec_url(api_url, namespace, pod_name, container["name"]), + subprotocols: ['channel.k8s.io'], + headers: Hash.new { |h, k| h[k] = [] }, + created_at: created_at, + } + end + end + + def add_terminal_auth(terminal, token, ca_pem = nil) + terminal[:headers]['Authorization'] << "Bearer #{token}" + terminal[:ca_pem] = ca_pem if ca_pem.present? + terminal + end + + def container_exec_url(api_url, namespace, pod_name, container_name) + url = URI.parse(api_url) + url.path = [ + url.path.sub(%r{/+\z}, ''), + 'api', 'v1', + 'namespaces', ERB::Util.url_encode(namespace), + 'pods', ERB::Util.url_encode(pod_name), + 'exec' + ].join('/') + + url.query = { + container: container_name, + tty: true, + stdin: true, + stdout: true, + stderr: true, + }.to_query + '&' + EXEC_COMMAND + + case url.scheme + when 'http' + url.scheme = 'ws' + when 'https' + url.scheme = 'wss' + end + + url.to_s + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index aeb1a26e1ba..d28bb583fe7 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -95,6 +95,19 @@ module Gitlab ] end + def terminal_websocket(terminal) + details = { + 'Terminal' => { + 'Subprotocols' => terminal[:subprotocols], + 'Url' => terminal[:url], + 'Header' => terminal[:headers] + } + } + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + + details + end + def version path = Rails.root.join(VERSION_FILE) path.readable? ? path.read.chomp : 'unknown' -- cgit v1.2.1 From 5652da8bb4ea26acd35a241683e242439fefdd33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 16 Dec 2016 18:58:24 +0100 Subject: Allow unauthenticated access to Repositories Files API GET endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/api/files.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/api/files.rb b/lib/api/files.rb index 28f306e45f3..532a317c89e 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,8 +1,6 @@ module API # Projects API class Files < Grape::API - before { authenticate! } - helpers do def commit_params(attrs) { -- cgit v1.2.1