From a2ef52b32bd3d1ee70ea333566e330c44e6c9170 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 15:36:41 -0300 Subject: Add custom OmniAuth strategy for Bitbucket OAuth2 --- lib/omniauth/strategies/bitbucket.rb | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 lib/omniauth/strategies/bitbucket.rb (limited to 'lib') diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb new file mode 100644 index 00000000000..2006e7582ce --- /dev/null +++ b/lib/omniauth/strategies/bitbucket.rb @@ -0,0 +1,45 @@ +require 'omniauth-oauth2' + +module OmniAuth + module Strategies + class Bitbucket < OmniAuth::Strategies::OAuth2 + option :name, 'bitbucket' + + option :client_options, { + site: 'https://bitbucket.org', + authorize_url: 'https://bitbucket.org/site/oauth2/authorize', + token_url: 'https://bitbucket.org/site/oauth2/access_token' + } + + def callback_url + full_host + script_name + callback_path + end + + uid do + raw['username'] + end + + info do + { + name: raw_info['display_name'], + avatar: raw_info['links']['avatar']['href'], + email: primary_email + } + end + + def raw_info + @raw_info ||= access_token.get('user').parsed + end + + def primary_email + primary = emails.find{ |i| i['is_primary'] && i['is_confirmed'] } + primary && primary['email'] || nil + end + + def emails + email_response = access_token.get('user/emails').parsed + @emails ||= email_response && email_response['values'] || nil + end + end + end +end -- cgit v1.2.1 From fc34c9560324b7db5bdaf205cbdf46a339102af2 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 15:42:26 -0300 Subject: Add a Bitbucket client for the OAuth2 API --- lib/bitbucket/client.rb | 11 ++++++++ lib/bitbucket/connection.rb | 69 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 lib/bitbucket/client.rb create mode 100644 lib/bitbucket/connection.rb (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb new file mode 100644 index 00000000000..5f48e52da98 --- /dev/null +++ b/lib/bitbucket/client.rb @@ -0,0 +1,11 @@ +module Bitbucket + class Client + def initialize(options = {}) + @connection = options.fetch(:connection, Connection.new(options)) + end + + private + + attr_reader :connection + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb new file mode 100644 index 00000000000..00f127f9507 --- /dev/null +++ b/lib/bitbucket/connection.rb @@ -0,0 +1,69 @@ +module Bitbucket + class Connection + DEFAULT_API_VERSION = '2.0' + DEFAULT_BASE_URI = 'https://api.bitbucket.org/' + DEFAULT_QUERY = {} + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @query = options.fetch(:query, DEFAULT_QUERY) + + @token = options.fetch(:token) + @expires_at = options.fetch(:expires_at) + @expires_in = options.fetch(:expires_in) + @refresh_token = options.fetch(:refresh_token) + + @client = OAuth2::Client.new(provider.app_id, provider.app_secret, options) + @connection = OAuth2::AccessToken.new(@client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + def query(params = {}) + @query.update(params) + end + + def get(path, query = {}) + refresh! if expired? + + response = connection.get(build_url(path), params: @query.merge(query)) + response.parsed + end + + def expired? + connection.expired? + end + + def refresh! + response = connection.refresh! + + @token = response.token + @expires_at = response.expires_at + @expires_in = response.expires_in + @refresh_token = response.refresh_token + + @connection = OAuth2::AccessToken.new(@client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + private + + attr_reader :connection, :expires_at, :expires_in, :refresh_token, :token + + def build_url(path) + return path if path.starts_with?(root_url) + + "#{root_url}#{path}" + end + + def root_url + @root_url ||= "#{@base_uri}#{@api_version}" + end + + def provider + Gitlab.config.omniauth.providers.find { |provider| provider.name == 'bitbucket' } + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys + end + end +end -- cgit v1.2.1 From f1f5863bfc5727dba4767a54a299b0cf526b025a Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 15:45:29 -0300 Subject: Add an endpoint to get the Bitbucket user profile --- lib/bitbucket/client.rb | 5 +++++ lib/bitbucket/representation/base.rb | 13 +++++++++++++ lib/bitbucket/representation/user.rb | 9 +++++++++ 3 files changed, 27 insertions(+) create mode 100644 lib/bitbucket/representation/base.rb create mode 100644 lib/bitbucket/representation/user.rb (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 5f48e52da98..c05fc35f36e 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -4,6 +4,11 @@ module Bitbucket @connection = options.fetch(:connection, Connection.new(options)) end + def user + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end + private attr_reader :connection diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb new file mode 100644 index 00000000000..7b639492d38 --- /dev/null +++ b/lib/bitbucket/representation/base.rb @@ -0,0 +1,13 @@ +module Bitbucket + module Representation + class Base + def initialize(raw) + @raw = raw + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb new file mode 100644 index 00000000000..ba6b7667b49 --- /dev/null +++ b/lib/bitbucket/representation/user.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class User < Representation::Base + def username + raw['username'] + end + end + end +end -- cgit v1.2.1 From e2f7f32a60a7663d12b5dae0320f640150f354e7 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 15:48:31 -0300 Subject: Add support for Bitbucket pagination when fetching collections --- lib/bitbucket/collection.rb | 21 +++++++++++++++++++++ lib/bitbucket/page.rb | 36 ++++++++++++++++++++++++++++++++++++ lib/bitbucket/paginator.rb | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 lib/bitbucket/collection.rb create mode 100644 lib/bitbucket/page.rb create mode 100644 lib/bitbucket/paginator.rb (limited to 'lib') diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb new file mode 100644 index 00000000000..9cc8947417c --- /dev/null +++ b/lib/bitbucket/collection.rb @@ -0,0 +1,21 @@ +module Bitbucket + class Collection < Enumerator + def initialize(paginator) + super() do |yielder| + loop do + paginator.next.each { |item| yielder << item } + end + end + + lazy + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.send(method, *args) do |item| + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb new file mode 100644 index 00000000000..ad9a2baba36 --- /dev/null +++ b/lib/bitbucket/page.rb @@ -0,0 +1,36 @@ +module Bitbucket + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + attrs.fetch(:next, false) + end + + def next + attrs.fetch(:next) + end + + private + + def parse_attrs(raw) + attrs = %w(size page pagelen next previous) + attrs.map { |attr| { attr.to_sym => raw[attr] } }.reduce(&:merge) + end + + def parse_values(raw, representation_class) + return [] if raw['values'].nil? || !raw['values'].is_a?(Array) + + raw['values'].map { |hash| representation_class.new(hash) } + end + + def representation_class(type) + class_name = "Bitbucket::Representation::#{type.to_s.camelize}" + class_name.constantize + end + end +end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb new file mode 100644 index 00000000000..a1672d9eaa1 --- /dev/null +++ b/lib/bitbucket/paginator.rb @@ -0,0 +1,38 @@ +module Bitbucket + class Paginator + PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. + + def initialize(connection, url, type) + @connection = connection + @type = type + @url = url + @page = nil + + connection.query(pagelen: PAGE_LENGTH, sort: :created_on) + end + + def next + raise StopIteration unless has_next_page? + + @page = fetch_next_page + @page.items + end + + private + + attr_reader :connection, :page, :url, :type + + def has_next_page? + page.nil? || page.next? + end + + def page_url + page.nil? ? url : page.next + end + + def fetch_next_page + parsed_response = connection.get(page_url) + Page.new(parsed_response, type) + end + end +end -- cgit v1.2.1 From 6418c6f88efe9015c8bc2ebd4f7db1a7277a4dc9 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 15:53:46 -0300 Subject: Add an endpoint to get the user's repositories --- lib/bitbucket/client.rb | 14 +++++++-- lib/bitbucket/representation/repo.rb | 57 ++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 lib/bitbucket/representation/repo.rb (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index c05fc35f36e..39b52ae25a6 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -4,9 +4,19 @@ module Bitbucket @connection = options.fetch(:connection, Connection.new(options)) end + + def repos + relative_path = "/repositories/#{user.username}" + paginator = Paginator.new(connection, relative_path, :repo) + + Collection.new(paginator) + end + def user - parsed_response = connection.get('/user') - Representation::User.new(parsed_response) + @user ||= begin + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end end private diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb new file mode 100644 index 00000000000..fe5cda66ab9 --- /dev/null +++ b/lib/bitbucket/representation/repo.rb @@ -0,0 +1,57 @@ +module Bitbucket + module Representation + class Repo < Representation::Base + attr_reader :owner, :slug + + def initialize(raw) + super(raw) + + if full_name && full_name.split('/').size == 2 + @owner, @slug = full_name.split('/') + end + end + + def clone_url(token = nil) + url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') + + if token.present? + url.sub(/^[^\@]*/, "https://x-token-auth:#{token}") + else + url + end + end + + def description + raw['description'] + end + + def full_name + raw['full_name'] + end + + def has_issues? + raw['has_issues'] + end + + def name + raw['name'] + end + + def valid? + raw['scm'] == 'git' + end + + def visibility_level + if raw['is_private'] + Gitlab::VisibilityLevel::PRIVATE + else + Gitlab::VisibilityLevel::PUBLIC + end + end + + def to_s + full_name + end + end + end +end -- cgit v1.2.1 From 411d0a93723467eb355155fe52f1d159d4527556 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 15:55:08 -0300 Subject: Add an endpoint to get a single user repository --- lib/bitbucket/client.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 39b52ae25a6..24984ca0793 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -5,6 +5,11 @@ module Bitbucket end + def repo(name) + parsed_response = connection.get("/repositories/#{name}") + Representation::Repo.new(parsed_response) + end + def repos relative_path = "/repositories/#{user.username}" paginator = Paginator.new(connection, relative_path, :repo) -- cgit v1.2.1 From 56cb4762d42f758ad6e4ec1874b7eed8e1c1f687 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:02:48 -0300 Subject: Refactoring Bitbucket import controller to use the new OAuth2 client --- lib/bitbucket/error/unauthorized.rb | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 lib/bitbucket/error/unauthorized.rb (limited to 'lib') diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb new file mode 100644 index 00000000000..5e2eb57bb0e --- /dev/null +++ b/lib/bitbucket/error/unauthorized.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Error + class Unauthorized < StandardError + end + end +end -- cgit v1.2.1 From ceac7878e9b5ba56f31f605c40095e8b83d83b6f Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:04:11 -0300 Subject: Clone Bitbucket repositories over HTTPS --- lib/gitlab/bitbucket_import/project_creator.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..9d80c5d4f2b 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -13,15 +13,15 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["slug"], - description: repo["description"], + name: repo.name, + path: repo.slug, + description: repo.description, namespace_id: namespace.id, - visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "bitbucket", - import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } + visibility_level: repo.visibility_level, + import_type: 'bitbucket', + import_source: repo.full_name, + import_url: repo.clone_url(@session_data[:token]), + import_data: { credentials: session_data } ).execute end end -- cgit v1.2.1 From 267e27b0cd543e8eeaa04686ad4678c4f553c479 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:05:44 -0300 Subject: Remove code to clone Bitbucket repositories using SSH --- lib/gitlab/bitbucket_import/key_adder.rb | 24 ------------------------ lib/gitlab/bitbucket_import/key_deleter.rb | 23 ----------------------- 2 files changed, 47 deletions(-) delete mode 100644 lib/gitlab/bitbucket_import/key_adder.rb delete mode 100644 lib/gitlab/bitbucket_import/key_deleter.rb (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb deleted file mode 100644 index 0b63f025d0a..00000000000 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyAdder - attr_reader :repo, :current_user, :client - - def initialize(repo, current_user, access_params) - @repo, @current_user = repo, current_user - @client = Client.new(access_params[:bitbucket_access_token], - access_params[:bitbucket_access_token_secret]) - end - - def execute - return false unless BitbucketImport.public_key.present? - - project_identifier = "#{repo["owner"]}/#{repo["slug"]}" - client.add_deploy_key(project_identifier, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb deleted file mode 100644 index e03c3155b3e..00000000000 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyDeleter - attr_reader :project, :current_user, :client - - def initialize(project) - @project = project - @current_user = project.creator - @client = Client.from_project(@project) - end - - def execute - return false unless BitbucketImport.public_key.present? - - client.delete_deploy_key(project.import_source, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end -- cgit v1.2.1 From 3dd15d3f753a5a71522275a37393bfa56d6e3517 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:09:25 -0300 Subject: Add an endpoint to get a list of issues for a repo --- lib/bitbucket/client.rb | 7 +++++ lib/bitbucket/representation/issue.rb | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 lib/bitbucket/representation/issue.rb (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 24984ca0793..ac6e91bb526 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -4,6 +4,13 @@ module Bitbucket @connection = options.fetch(:connection, Connection.new(options)) end + def issues(repo) + relative_path = "/repositories/#{repo}/issues" + paginator = Paginator.new(connection, relative_path, :issue) + + Collection.new(paginator) + end + def repo(name) parsed_response = connection.get("/repositories/#{name}") diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb new file mode 100644 index 00000000000..48647ad51f6 --- /dev/null +++ b/lib/bitbucket/representation/issue.rb @@ -0,0 +1,49 @@ +module Bitbucket + module Representation + class Issue < Representation::Base + CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze + + def iid + raw['id'] + end + + def author + reporter.fetch('username', 'Anonymous') + end + + def description + raw.dig('content', 'raw') + end + + def state + closed? ? 'closed' : 'opened' + end + + def title + raw['title'] + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['edited_on'] + end + + def to_s + iid + end + + private + + def closed? + CLOSED_STATUS.include?(raw['state']) + end + + def reporter + raw.fetch('reporter', {}) + end + end + end +end -- cgit v1.2.1 From 3f59d25d263d1ac9db76cd2d3d4d025fb6d6dff4 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:10:29 -0300 Subject: Add an endpoint to get a list of issue comments --- lib/bitbucket/client.rb | 10 ++++++++++ lib/bitbucket/representation/comment.rb | 27 +++++++++++++++++++++++++++ lib/bitbucket/representation/url.rb | 9 +++++++++ 3 files changed, 46 insertions(+) create mode 100644 lib/bitbucket/representation/comment.rb create mode 100644 lib/bitbucket/representation/url.rb (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index ac6e91bb526..3d22347603d 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -11,6 +11,16 @@ module Bitbucket Collection.new(paginator) end + def issue_comments(repo, number) + relative_path = "/repositories/#{repo}/issues/#{number}/comments" + paginator = Paginator.new(connection, relative_path, :url) + + Collection.new(paginator).map do |comment_url| + parsed_response = connection.get(comment_url.to_s) + Representation::Comment.new(parsed_response) + end + end + def repo(name) parsed_response = connection.get("/repositories/#{name}") diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb new file mode 100644 index 00000000000..94bc18cbfab --- /dev/null +++ b/lib/bitbucket/representation/comment.rb @@ -0,0 +1,27 @@ +module Bitbucket + module Representation + class Comment < Representation::Base + def author + user.fetch('username', 'Anonymous') + end + + def note + raw.dig('content', 'raw') + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] || raw['created_on'] + end + + private + + def user + raw.fetch('user', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/url.rb b/lib/bitbucket/representation/url.rb new file mode 100644 index 00000000000..24ae1048013 --- /dev/null +++ b/lib/bitbucket/representation/url.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class Url < Representation::Base + def to_s + raw.dig('links', 'self', 'href') + end + end + end +end -- cgit v1.2.1 From 3c756b83ef04dbbb2a82a53cf785a87da0772255 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:11:45 -0300 Subject: Remove client for the Bitbucket API version 1.0 --- lib/gitlab/bitbucket_import/client.rb | 142 ---------------------------------- 1 file changed, 142 deletions(-) delete mode 100644 lib/gitlab/bitbucket_import/client.rb (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb deleted file mode 100644 index 8d1ad62fae0..00000000000 --- a/lib/gitlab/bitbucket_import/client.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module BitbucketImport - class Client - class Unauthorized < StandardError; end - - attr_reader :consumer, :api - - def self.from_project(project) - import_data_credentials = project.import_data.credentials if project.import_data - if import_data_credentials && import_data_credentials[:bb_session] - token = import_data_credentials[:bb_session][:bitbucket_access_token] - token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] - new(token, token_secret) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" - end - end - - def initialize(access_token = nil, access_token_secret = nil) - @consumer = ::OAuth::Consumer.new( - config.app_id, - config.app_secret, - bitbucket_options - ) - - if access_token && access_token_secret - @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) - end - end - - def request_token(redirect_uri) - request_token = consumer.get_request_token(oauth_callback: redirect_uri) - - { - oauth_token: request_token.token, - oauth_token_secret: request_token.secret, - oauth_callback_confirmed: request_token.callback_confirmed?.to_s - } - end - - def authorize_url(request_token, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.authorize_url - else - request_token.authorize_url(oauth_callback: redirect_uri) - end - end - - def get_token(request_token, oauth_verifier, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.get_access_token(oauth_verifier: oauth_verifier) - else - request_token.get_access_token(oauth_callback: redirect_uri) - end - end - - def user - JSON.parse(get("/api/1.0/user").body) - end - - def issues(project_identifier) - all_issues = [] - offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket - index = 0 - - begin - issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) - # Find out how many total issues are present - total = issues["count"] if index == 0 - all_issues.concat(issues["issues"]) - offset += issues["issues"].count - index += 1 - end while all_issues.count < total - - all_issues - end - - def issue_comments(project_identifier, issue_id) - comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) - comments.sort_by { |comment| comment["utc_created_on"] } - end - - def project(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body) - end - - def find_deploy_key(project_identifier, key) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| - deploy_key["key"].chomp == key.chomp - end - end - - def add_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return if deploy_key - - JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) - end - - def delete_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return unless deploy_key - - api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" - end - - def projects - JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } - end - - def incompatible_projects - JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" } - end - - private - - def get(url) - response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) - - response - end - - def issue_api_endpoint(project_identifier, per_page, offset) - "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" - end - - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } - end - - def bitbucket_options - OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys - end - end - end -end -- cgit v1.2.1 From 317b020932736d2cd629542e3a8b3aef2219e033 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:13:56 -0300 Subject: Refactoring Bitbucket importer to use the new OAuth2 client --- lib/gitlab/bitbucket_import/importer.rb | 74 +++++++++++++++------------------ 1 file changed, 33 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4b5097adb1..67e906431f0 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -5,18 +5,14 @@ module Gitlab def initialize(project) @project = project - @client = Client.from_project(@project) + @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new end def execute - import_issues if has_issues? + import_issues true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error.new, e.message - ensure - Gitlab::BitbucketImport::KeyDeleter.new(project).execute end private @@ -30,44 +26,40 @@ module Gitlab end end - def identifier - project.import_source - end - - def has_issues? - client.project(identifier)["has_issues"] + def repo + @repo ||= client.repo(project.import_source) end def import_issues - issues = client.issues(identifier) - - issues.each do |issue| - body = '' - reporter = nil - author = 'Anonymous' - - if issue["reported_by"] && issue["reported_by"]["username"] - reporter = issue["reported_by"]["username"] - author = reporter - end - - body = @formatter.author_line(author) - body += issue["content"] - - comments = client.issue_comments(identifier, issue["local_id"]) - - if comments.any? - body += @formatter.comments_header - end - - comments.each do |comment| - author = 'Anonymous' + return unless repo.has_issues? + + client.issues(repo).each do |issue| + description = @formatter.author_line(issue.author) + description += issue.description + + issue = project.issues.create( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gl_user_id(project, issue.author), + created_at: issue.created_at, + updated_at: issue.updated_at + ) - if comment["author_info"] && comment["author_info"]["username"] - author = comment["author_info"]["username"] + if issue.persisted? + client.issue_comments(repo, issue.iid).each do |comment| + note = @formatter.author_line(comment.author) + note += comment.note + + issue.notes.create!( + project: project, + note: note, + author_id: gl_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) end - - body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) end project.issues.create!( @@ -77,8 +69,8 @@ module Gitlab author_id: gitlab_user_id(project, reporter) ) end - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + rescue ActiveRecord::RecordInvalid + nil end end end -- cgit v1.2.1 From 64722a15e39436820a0636804179cf8c8957197e Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:15:15 -0300 Subject: Add an endpoint to get a list of pull requests for a repo --- lib/bitbucket/client.rb | 6 ++++++ lib/bitbucket/representation/pull_request.rb | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 lib/bitbucket/representation/pull_request.rb (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 3d22347603d..0368f388ea4 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -21,6 +21,12 @@ module Bitbucket end end + def pull_requests(repo) + relative_path = "/repositories/#{repo}/pullrequests" + paginator = Paginator.new(connection, relative_path, :pull_request) + + Collection.new(paginator) + end def repo(name) parsed_response = connection.get("/repositories/#{name}") diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb new file mode 100644 index 00000000000..7cbad91e9c8 --- /dev/null +++ b/lib/bitbucket/representation/pull_request.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Representation + class PullRequest < Representation::Base + end + end +end -- cgit v1.2.1 From 704115c726999d6f0467bbf70087db3ae690d3ab Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 22 Aug 2016 16:15:39 -0300 Subject: Import opened pull request from Bitbucket --- lib/bitbucket/representation/pull_request.rb | 59 ++++++++++++++++++++++++++++ lib/gitlab/bitbucket_import/importer.rb | 31 +++++++++++++++ 2 files changed, 90 insertions(+) (limited to 'lib') diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index 7cbad91e9c8..e7b1f99e9a6 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -1,6 +1,65 @@ module Bitbucket module Representation class PullRequest < Representation::Base + def author + raw.fetch('author', {}).fetch('username', 'Anonymous') + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + if raw['state'] == 'MERGED' + 'merged' + elsif raw['state'] == 'DECLINED' + 'closed' + else + 'opened' + end + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] + end + + def title + raw['title'] + end + + def source_branch_name + source_branch.dig('branch', 'name') + end + + def source_branch_sha + source_branch.dig('commit', 'hash') + end + + def target_branch_name + target_branch.dig('branch', 'name') + end + + def target_branch_sha + target_branch.dig('commit', 'hash') + end + + private + + def source_branch + raw['source'] + end + + def target_branch + raw['destination'] + end end end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 67e906431f0..0060e350249 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -11,6 +11,7 @@ module Gitlab def execute import_issues + import_pull_requests true end @@ -72,6 +73,36 @@ module Gitlab rescue ActiveRecord::RecordInvalid nil end + + def import_pull_requests + pull_requests = client.pull_requests(repo) + + pull_requests.each do |pull_request| + begin + description = @formatter.author_line(pull_request.author) + description += pull_request.description + + project.merge_requests.create( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: gl_user_id(project, pull_request.author), + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + rescue ActiveRecord::RecordInvalid + nil + end + end + end end end end -- cgit v1.2.1 From 9860488360681a3b10c3de04606ef931c3639601 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 11 Nov 2016 16:07:16 -0800 Subject: Fix missing Bitbucket ProjectController changes for specifying namespace/project --- lib/gitlab/bitbucket_import/project_creator.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 9d80c5d4f2b..b34be272af3 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,8 +14,8 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo.name, - path: repo.slug, + name: name, + path: name, description: repo.description, namespace_id: namespace.id, visibility_level: repo.visibility_level, -- cgit v1.2.1 From b25ebfe67b0e5448e9625e7a5c469ab41a4b7059 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 11 Nov 2016 16:08:03 -0800 Subject: Lazily load Bitbucket connection --- lib/bitbucket/connection.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index 00f127f9507..e9cbf36a44b 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -13,13 +13,18 @@ module Bitbucket @expires_at = options.fetch(:expires_at) @expires_in = options.fetch(:expires_in) @refresh_token = options.fetch(:refresh_token) + end - @client = OAuth2::Client.new(provider.app_id, provider.app_secret, options) - @connection = OAuth2::AccessToken.new(@client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) end def query(params = {}) - @query.update(params) + @query.merge!(params) end def get(path, query = {}) @@ -46,7 +51,7 @@ module Bitbucket private - attr_reader :connection, :expires_at, :expires_in, :refresh_token, :token + attr_reader :expires_at, :expires_in, :refresh_token, :token def build_url(path) return path if path.starts_with?(root_url) -- cgit v1.2.1 From 3b9d1c0f5d2996b946ab71704995b752c0ff8f60 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 11 Nov 2016 16:10:12 -0800 Subject: Fix refresh to lazily load connection --- lib/bitbucket/connection.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index e9cbf36a44b..201e7e3b808 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -45,8 +45,7 @@ module Bitbucket @expires_at = response.expires_at @expires_in = response.expires_in @refresh_token = response.refresh_token - - @connection = OAuth2::AccessToken.new(@client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + @connection = nil end private -- cgit v1.2.1 From 82d7a3a3dd61c70c87a8a4a116b2ce6c0612de59 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 11 Nov 2016 16:33:28 -0800 Subject: Fix typos in pull requests failing to import --- lib/gitlab/bitbucket_import/importer.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 0060e350249..15cc64dd7f4 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -43,7 +43,7 @@ module Gitlab title: issue.title, description: description, state: issue.state, - author_id: gl_user_id(project, issue.author), + author_id: gitlab_user_id(project, issue.author), created_at: issue.created_at, updated_at: issue.updated_at ) @@ -56,7 +56,7 @@ module Gitlab issue.notes.create!( project: project, note: note, - author_id: gl_user_id(project, comment.author), + author_id: gitlab_user_id(project, comment.author), created_at: comment.created_at, updated_at: comment.updated_at ) @@ -93,7 +93,7 @@ module Gitlab target_branch: pull_request.target_branch_name, target_branch_sha: pull_request.target_branch_sha, state: pull_request.state, - author_id: gl_user_id(project, pull_request.author), + author_id: gitlab_user_id(project, pull_request.author), assignee_id: nil, created_at: pull_request.created_at, updated_at: pull_request.updated_at -- cgit v1.2.1 From 489d241c8d68ed527fccb73a1f7e46e9a567c971 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 11 Nov 2016 16:48:04 -0800 Subject: Incorporate review comments --- lib/bitbucket/page.rb | 3 +-- lib/omniauth/strategies/bitbucket.rb | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb index ad9a2baba36..0e8ce11aa1d 100644 --- a/lib/bitbucket/page.rb +++ b/lib/bitbucket/page.rb @@ -29,8 +29,7 @@ module Bitbucket end def representation_class(type) - class_name = "Bitbucket::Representation::#{type.to_s.camelize}" - class_name.constantize + class_name = Bitbucket::Representation.const_get(type.to_s.camelize) end end end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb index 2006e7582ce..c5484c59c47 100644 --- a/lib/omniauth/strategies/bitbucket.rb +++ b/lib/omniauth/strategies/bitbucket.rb @@ -32,7 +32,7 @@ module OmniAuth end def primary_email - primary = emails.find{ |i| i['is_primary'] && i['is_confirmed'] } + primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] } primary && primary['email'] || nil end -- cgit v1.2.1 From ea393e6f308e5dcdd5c48433285594db0539b203 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 16 Nov 2016 00:13:17 -0800 Subject: Import pull request comments --- lib/bitbucket/client.rb | 7 +++ .../representation/pull_request_comment.rb | 39 ++++++++++++ lib/gitlab/bitbucket_import/importer.rb | 70 +++++++++++++++++++++- 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 lib/bitbucket/representation/pull_request_comment.rb (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 0368f388ea4..fce1c898030 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -28,6 +28,13 @@ module Bitbucket Collection.new(paginator) end + def pull_request_comments(repo, pull_request) + relative_path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + paginator = Paginator.new(connection, relative_path, :pull_request_comment) + + Collection.new(paginator) + end + def repo(name) parsed_response = connection.get("/repositories/#{name}") Representation::Repo.new(parsed_response) diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb new file mode 100644 index 00000000000..94719edbf38 --- /dev/null +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -0,0 +1,39 @@ +module Bitbucket + module Representation + class PullRequestComment < Comment + def iid + raw['id'] + end + + def file_path + inline.fetch('path', nil) + end + + def old_pos + inline.fetch('from', nil) || 1 + end + + def new_pos + inline.fetch('to', nil) || 1 + end + + def parent_id + raw.fetch('parent', {}).fetch('id', nil) + end + + def inline? + raw.has_key?('inline') + end + + def has_parent? + raw.has_key?('parent') + end + + private + + def inline + raw.fetch('inline', {}) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 15cc64dd7f4..94e8062e447 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -82,7 +82,7 @@ module Gitlab description = @formatter.author_line(pull_request.author) description += pull_request.description - project.merge_requests.create( + merge_request = project.merge_requests.create( iid: pull_request.iid, title: pull_request.title, description: description, @@ -98,11 +98,79 @@ module Gitlab created_at: pull_request.created_at, updated_at: pull_request.updated_at ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? rescue ActiveRecord::RecordInvalid nil end end end + + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) + + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end + + def import_inline_comments(inline_comments, pull_request, merge_request) + line_code_map = {} + + children, parents = inline_comments.partition(&:has_parent?) + + # The Bitbucket API returns threaded replies as parent-child + # relationships. We assume that the child can appear in any order in + # the JSON. + parents.each do |comment| + line_code_map[comment.iid] = generate_line_code(comment) + end + + children.each do |comment| + line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + end + + inline_comments.each do |comment| + begin + attributes = pull_request_comment_attributes(comment) + attributes.merge!( + commit_id: pull_request.source_branch_sha, + line_code: line_code_map.fetch(comment.iid), + type: 'LegacyDiffNote') + + note = merge_request.notes.create!(attributes) + rescue ActiveRecord::RecordInvalid => e + Rails.log.error("Bitbucket importer ERROR: Invalid pull request comment #{e.message}") + nil + end + end + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + rescue ActiveRecord::RecordInvalid => e + Rails.log.error("Bitbucket importer ERROR: Invalid standalone pull request comment #{e.message}") + nil + end + end + end + + def generate_line_code(pr_comment) + Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + end + + def pull_request_comment_attributes(comment) + { + project: project, + note: comment.note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + } + end end end end -- cgit v1.2.1 From b8bf28348fb903c62e084353896873438f4f0845 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 17 Nov 2016 20:16:22 -0800 Subject: Rubocop fixes --- lib/bitbucket/page.rb | 2 +- lib/gitlab/bitbucket_import/importer.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb index 0e8ce11aa1d..bc51ce7dce2 100644 --- a/lib/bitbucket/page.rb +++ b/lib/bitbucket/page.rb @@ -29,7 +29,7 @@ module Bitbucket end def representation_class(type) - class_name = Bitbucket::Representation.const_get(type.to_s.camelize) + Bitbucket::Representation.const_get(type.to_s.camelize) end end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 94e8062e447..1f7a691e6dd 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -139,7 +139,7 @@ module Gitlab line_code: line_code_map.fetch(comment.iid), type: 'LegacyDiffNote') - note = merge_request.notes.create!(attributes) + merge_request.notes.create!(attributes) rescue ActiveRecord::RecordInvalid => e Rails.log.error("Bitbucket importer ERROR: Invalid pull request comment #{e.message}") nil -- cgit v1.2.1 From 9e6b25d0bc5c2f88330bb074db242017ea45f90d Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 17 Nov 2016 21:30:35 -0800 Subject: Add support for extracting all pull requests and their raw diffs --- lib/bitbucket/client.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index fce1c898030..0d4cfd600b8 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -22,7 +22,7 @@ module Bitbucket end def pull_requests(repo) - relative_path = "/repositories/#{repo}/pullrequests" + relative_path = "/repositories/#{repo}/pullrequests?state=ALL" paginator = Paginator.new(connection, relative_path, :pull_request) Collection.new(paginator) @@ -35,6 +35,12 @@ module Bitbucket Collection.new(paginator) end + def pull_request_diff(repo, pull_request) + relative_path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + + connection.get(relative_path) + end + def repo(name) parsed_response = connection.get("/repositories/#{name}") Representation::Repo.new(parsed_response) -- cgit v1.2.1 From 4d7303a98e970c29079cc03a449c71f3cdaa1214 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 18 Nov 2016 21:35:03 -0800 Subject: Clean up owner and slug representation --- lib/bitbucket/representation/repo.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb index fe5cda66ab9..b291dfe0441 100644 --- a/lib/bitbucket/representation/repo.rb +++ b/lib/bitbucket/representation/repo.rb @@ -5,10 +5,18 @@ module Bitbucket def initialize(raw) super(raw) + end - if full_name && full_name.split('/').size == 2 - @owner, @slug = full_name.split('/') - end + def owner_and_slug + @owner_and_slug ||= full_name.split('/', 2) + end + + def owner + owner_and_slug.first + end + + def slug + owner_and_slug.last end def clone_url(token = nil) -- cgit v1.2.1 From c7c4d657b427c6fa146319ccc5aa17e87d3d0e0b Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 19 Nov 2016 21:44:19 -0800 Subject: Clean up Bitbucket connection based on review comments --- lib/bitbucket/client.rb | 2 +- lib/bitbucket/connection.rb | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 0d4cfd600b8..33e977d655d 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -1,7 +1,7 @@ module Bitbucket class Client def initialize(options = {}) - @connection = options.fetch(:connection, Connection.new(options)) + @connection = Connection.new(options) end def issues(repo) diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index 201e7e3b808..c375fe16aee 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -7,12 +7,12 @@ module Bitbucket def initialize(options = {}) @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) - @query = options.fetch(:query, DEFAULT_QUERY) + @default_query = options.fetch(:query, DEFAULT_QUERY) - @token = options.fetch(:token) - @expires_at = options.fetch(:expires_at) - @expires_in = options.fetch(:expires_in) - @refresh_token = options.fetch(:refresh_token) + @token = options[:token] + @expires_at = options[:expires_at] + @expires_in = options[:expires_in] + @refresh_token = options[:refresh_token] end def client @@ -24,13 +24,13 @@ module Bitbucket end def query(params = {}) - @query.merge!(params) + @default_query.merge!(params) end - def get(path, query = {}) + def get(path, extra_query = {}) refresh! if expired? - response = connection.get(build_url(path), params: @query.merge(query)) + response = connection.get(build_url(path), params: @default_query.merge(extra_query)) response.parsed end -- cgit v1.2.1 From 0b72994b63dbbbddaf0e77629249d92890a6e4a4 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 21 Nov 2016 08:25:47 -0800 Subject: Simplify Bitbucket::Page implementation --- lib/bitbucket/page.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb index bc51ce7dce2..b91a173b53c 100644 --- a/lib/bitbucket/page.rb +++ b/lib/bitbucket/page.rb @@ -23,7 +23,7 @@ module Bitbucket end def parse_values(raw, representation_class) - return [] if raw['values'].nil? || !raw['values'].is_a?(Array) + return [] unless raw['values'] && raw['values'].is_a?(Array) raw['values'].map { |hash| representation_class.new(hash) } end -- cgit v1.2.1 From 402cc95c1a1df8168467f74e21c6df7d48359714 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 21 Nov 2016 20:49:40 -0800 Subject: Fix Bitbucket importer spec to pass with 2.0 API --- lib/bitbucket/page.rb | 4 ++-- lib/gitlab/bitbucket_import/importer.rb | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb index b91a173b53c..49d083cc66f 100644 --- a/lib/bitbucket/page.rb +++ b/lib/bitbucket/page.rb @@ -22,10 +22,10 @@ module Bitbucket attrs.map { |attr| { attr.to_sym => raw[attr] } }.reduce(&:merge) end - def parse_values(raw, representation_class) + def parse_values(raw, bitbucket_rep_class) return [] unless raw['values'] && raw['values'].is_a?(Array) - raw['values'].map { |hash| representation_class.new(hash) } + raw['values'].map { |hash| bitbucket_rep_class.new(hash) } end def representation_class(type) diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 1f7a691e6dd..729b465e861 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -62,13 +62,6 @@ module Gitlab ) end end - - project.issues.create!( - description: body, - title: issue["title"], - state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gitlab_user_id(project, reporter) - ) end rescue ActiveRecord::RecordInvalid nil -- cgit v1.2.1 From 7a155137a4fd965cb8ff512a9548a7e685b330f5 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 21 Nov 2016 22:06:09 -0800 Subject: Fix spec for Bitbucket importer --- lib/gitlab/bitbucket_import/importer.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 729b465e861..08705afcabb 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -94,7 +94,7 @@ module Gitlab import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? rescue ActiveRecord::RecordInvalid - nil + Rails.log.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request #{e.message}") end end end @@ -134,7 +134,7 @@ module Gitlab merge_request.notes.create!(attributes) rescue ActiveRecord::RecordInvalid => e - Rails.log.error("Bitbucket importer ERROR: Invalid pull request comment #{e.message}") + Rails.log.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request comment #{e.message}") nil end end @@ -145,7 +145,7 @@ module Gitlab begin merge_request.notes.create!(pull_request_comment_attributes(comment)) rescue ActiveRecord::RecordInvalid => e - Rails.log.error("Bitbucket importer ERROR: Invalid standalone pull request comment #{e.message}") + Rails.log.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid standalone pull request comment #{e.message}") nil end end -- cgit v1.2.1 From 101cde38cf6d5506ea37c5f912fb4c37af50c541 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Thu, 24 Nov 2016 17:00:37 +0800 Subject: Use Ci::Pipeline#latest for finding pipelines Feedback: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7333#note_18861407 --- lib/gitlab/badge/build/status.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb index 50aa45e5406..f78dfd5b83a 100644 --- a/lib/gitlab/badge/build/status.rb +++ b/lib/gitlab/badge/build/status.rb @@ -20,7 +20,8 @@ module Gitlab def status @project.pipelines - .where(sha: @sha, ref: @ref) + .where(sha: @sha) + .latest(@ref) .status || 'unknown' end -- cgit v1.2.1 From 54221b5a3b9a2489f979944c77298c4adf004984 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 5 Dec 2016 20:34:11 +0200 Subject: Fix inline comment importing for 1:1 diff type --- .../representation/pull_request_comment.rb | 2 +- lib/gitlab/bitbucket_import/importer.rb | 26 ++++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index 94719edbf38..38090188919 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -14,7 +14,7 @@ module Bitbucket end def new_pos - inline.fetch('to', nil) || 1 + inline.fetch('to', nil) || old_pos || 1 end def parent_id diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 08705afcabb..6438c8a52e4 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -81,10 +81,10 @@ module Gitlab description: description, source_project: project, source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, + source_branch_sha: project.repository.rugged.lookup(pull_request.source_branch_sha).oid, target_project: project, target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, + target_branch_sha: project.repository.rugged.lookup(pull_request.target_branch_sha).oid, state: pull_request.state, author_id: gitlab_user_id(project, pull_request.author), assignee_id: nil, @@ -94,7 +94,7 @@ module Gitlab import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? rescue ActiveRecord::RecordInvalid - Rails.log.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request #{e.message}") + Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request #{e.message}") end end end @@ -128,24 +128,36 @@ module Gitlab begin attributes = pull_request_comment_attributes(comment) attributes.merge!( - commit_id: pull_request.source_branch_sha, + position: build_position(merge_request, comment), line_code: line_code_map.fetch(comment.iid), - type: 'LegacyDiffNote') + type: 'DiffNote') merge_request.notes.create!(attributes) rescue ActiveRecord::RecordInvalid => e - Rails.log.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request comment #{e.message}") + Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request comment #{e.message}") nil end end end + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + def import_standalone_pr_comments(pr_comments, merge_request) pr_comments.each do |comment| begin merge_request.notes.create!(pull_request_comment_attributes(comment)) rescue ActiveRecord::RecordInvalid => e - Rails.log.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid standalone pull request comment #{e.message}") + Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid standalone pull request comment #{e.message}") nil end end -- cgit v1.2.1 From 84f2c219aa33de4890c7681372dd03309f216795 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 6 Dec 2016 13:46:59 +0200 Subject: Fix importing inline comment for any diff type --- lib/bitbucket/representation/pull_request_comment.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index 38090188919..c63d749cba7 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -10,11 +10,11 @@ module Bitbucket end def old_pos - inline.fetch('from', nil) || 1 + inline.fetch('from', nil) end def new_pos - inline.fetch('to', nil) || old_pos || 1 + inline.fetch('to', nil) end def parent_id -- cgit v1.2.1 From ee8433466ee77c1da026842e965b32ebefab6f13 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 6 Dec 2016 17:12:11 +0200 Subject: Fix importing PRs with not existing branches --- lib/bitbucket/connection.rb | 2 +- lib/gitlab/bitbucket_import/importer.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index c375fe16aee..e28285f119c 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -45,7 +45,7 @@ module Bitbucket @expires_at = response.expires_at @expires_in = response.expires_in @refresh_token = response.refresh_token - @connection = nil + @connection = nil end private diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 6438c8a52e4..34d93542955 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -81,10 +81,10 @@ module Gitlab description: description, source_project: project, source_branch: pull_request.source_branch_name, - source_branch_sha: project.repository.rugged.lookup(pull_request.source_branch_sha).oid, + source_branch_sha: pull_request.source_branch_sha, target_project: project, target_branch: pull_request.target_branch_name, - target_branch_sha: project.repository.rugged.lookup(pull_request.target_branch_sha).oid, + target_branch_sha: pull_request.target_branch_sha, state: pull_request.state, author_id: gitlab_user_id(project, pull_request.author), assignee_id: nil, -- cgit v1.2.1 From 43b7b0ce23d4de7055dc1cdd660b92ff03f4eb1e Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 6 Dec 2016 18:26:24 +0200 Subject: Fix authorization with BitBucket --- lib/omniauth/strategies/bitbucket.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb index c5484c59c47..475aad5970f 100644 --- a/lib/omniauth/strategies/bitbucket.rb +++ b/lib/omniauth/strategies/bitbucket.rb @@ -16,7 +16,7 @@ module OmniAuth end uid do - raw['username'] + raw_info['username'] end info do -- cgit v1.2.1 From 67b7637e5d7d3cf3e3f5cde6e7f984ece368c48c Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 7 Dec 2016 11:33:32 +0200 Subject: Apply review comments. Iteration 1 --- lib/bitbucket/client.rb | 29 +++++++++++----------- lib/bitbucket/connection.rb | 8 +++--- lib/bitbucket/paginator.rb | 2 +- lib/bitbucket/representation/issue.rb | 6 +---- .../representation/pull_request_comment.rb | 2 +- lib/bitbucket/representation/repo.rb | 6 +++-- lib/gitlab/bitbucket_import/importer.rb | 2 +- 7 files changed, 26 insertions(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 33e977d655d..9fa44506374 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -5,40 +5,39 @@ module Bitbucket end def issues(repo) - relative_path = "/repositories/#{repo}/issues" - paginator = Paginator.new(connection, relative_path, :issue) + path = "/repositories/#{repo}/issues" + paginator = Paginator.new(connection, path, :issue) Collection.new(paginator) end - def issue_comments(repo, number) - relative_path = "/repositories/#{repo}/issues/#{number}/comments" - paginator = Paginator.new(connection, relative_path, :url) + def issue_comments(repo, issue_id) + path = "/repositories/#{repo}/issues/#{issue_id}/comments" + paginator = Paginator.new(connection, path, :url) Collection.new(paginator).map do |comment_url| - parsed_response = connection.get(comment_url.to_s) - Representation::Comment.new(parsed_response) + Representation::Comment.new(connection.get(comment_url.to_s)) end end def pull_requests(repo) - relative_path = "/repositories/#{repo}/pullrequests?state=ALL" - paginator = Paginator.new(connection, relative_path, :pull_request) + path = "/repositories/#{repo}/pullrequests?state=ALL" + paginator = Paginator.new(connection, path, :pull_request) Collection.new(paginator) end def pull_request_comments(repo, pull_request) - relative_path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" - paginator = Paginator.new(connection, relative_path, :pull_request_comment) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + paginator = Paginator.new(connection, path, :pull_request_comment) Collection.new(paginator) end def pull_request_diff(repo, pull_request) - relative_path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" - connection.get(relative_path) + connection.get(path) end def repo(name) @@ -47,8 +46,8 @@ module Bitbucket end def repos - relative_path = "/repositories/#{user.username}" - paginator = Paginator.new(connection, relative_path, :repo) + path = "/repositories/#{user.username}" + paginator = Paginator.new(connection, path, :repo) Collection.new(paginator) end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index e28285f119c..692a596c057 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -5,8 +5,8 @@ module Bitbucket DEFAULT_QUERY = {} def initialize(options = {}) - @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) - @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) @default_query = options.fetch(:query, DEFAULT_QUERY) @token = options[:token] @@ -23,7 +23,7 @@ module Bitbucket @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) end - def query(params = {}) + def set_default_query_parameters(params = {}) @default_query.merge!(params) end @@ -63,7 +63,7 @@ module Bitbucket end def provider - Gitlab.config.omniauth.providers.find { |provider| provider.name == 'bitbucket' } + Gitlab::OAuth::Provider.config_for('bitbucket') end def options diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb index a1672d9eaa1..d0e23007ff8 100644 --- a/lib/bitbucket/paginator.rb +++ b/lib/bitbucket/paginator.rb @@ -8,7 +8,7 @@ module Bitbucket @url = url @page = nil - connection.query(pagelen: PAGE_LENGTH, sort: :created_on) + connection.set_default_query_parameters(pagelen: PAGE_LENGTH, sort: :created_on) end def next diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index 48647ad51f6..dc034c19750 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -8,7 +8,7 @@ module Bitbucket end def author - reporter.fetch('username', 'Anonymous') + raw.dig('reporter', 'username') || 'Anonymous' end def description @@ -40,10 +40,6 @@ module Bitbucket def closed? CLOSED_STATUS.include?(raw['state']) end - - def reporter - raw.fetch('reporter', {}) - end end end end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index c63d749cba7..ae2b069d6a2 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -18,7 +18,7 @@ module Bitbucket end def parent_id - raw.fetch('parent', {}).fetch('id', nil) + raw.dig('parent', 'id') end def inline? diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb index b291dfe0441..8969ecd1c19 100644 --- a/lib/bitbucket/representation/repo.rb +++ b/lib/bitbucket/representation/repo.rb @@ -23,7 +23,9 @@ module Bitbucket url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') if token.present? - url.sub(/^[^\@]*/, "https://x-token-auth:#{token}") + clone_url = URI::parse(url) + clone_url.user = "x-token-auth:#{token}" + clone_url.to_s else url end @@ -37,7 +39,7 @@ module Bitbucket raw['full_name'] end - def has_issues? + def issues_enabled? raw['has_issues'] end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 34d93542955..0f583b07e93 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -32,7 +32,7 @@ module Gitlab end def import_issues - return unless repo.has_issues? + return unless repo.issues_enabled? client.issues(repo).each do |issue| description = @formatter.author_line(issue.author) -- cgit v1.2.1 From b12d6541835024eb74384551b84bf0e74747d0c3 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 7 Dec 2016 14:00:06 +0200 Subject: BitBuckpet importer. Refactoring. Iteration 2 --- lib/bitbucket/client.rb | 6 ++---- lib/bitbucket/page.rb | 6 ++---- lib/bitbucket/paginator.rb | 4 ++-- lib/bitbucket/representation/base.rb | 4 ++++ lib/bitbucket/representation/url.rb | 9 --------- lib/gitlab/bitbucket_import/importer.rb | 7 +++++++ 6 files changed, 17 insertions(+), 19 deletions(-) delete mode 100644 lib/bitbucket/representation/url.rb (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 9fa44506374..3457c2c6454 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -13,11 +13,9 @@ module Bitbucket def issue_comments(repo, issue_id) path = "/repositories/#{repo}/issues/#{issue_id}/comments" - paginator = Paginator.new(connection, path, :url) + paginator = Paginator.new(connection, path, :comment) - Collection.new(paginator).map do |comment_url| - Representation::Comment.new(connection.get(comment_url.to_s)) - end + Collection.new(paginator) end def pull_requests(repo) diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb index 49d083cc66f..8f50f67f84d 100644 --- a/lib/bitbucket/page.rb +++ b/lib/bitbucket/page.rb @@ -18,14 +18,12 @@ module Bitbucket private def parse_attrs(raw) - attrs = %w(size page pagelen next previous) - attrs.map { |attr| { attr.to_sym => raw[attr] } }.reduce(&:merge) + raw.slice(*%w(size page pagelen next previous)).symbolize_keys end def parse_values(raw, bitbucket_rep_class) return [] unless raw['values'] && raw['values'].is_a?(Array) - - raw['values'].map { |hash| bitbucket_rep_class.new(hash) } + bitbucket_rep_class.decorate(raw['values']) end def representation_class(type) diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb index d0e23007ff8..37f12328447 100644 --- a/lib/bitbucket/paginator.rb +++ b/lib/bitbucket/paginator.rb @@ -26,12 +26,12 @@ module Bitbucket page.nil? || page.next? end - def page_url + def next_url page.nil? ? url : page.next end def fetch_next_page - parsed_response = connection.get(page_url) + parsed_response = connection.get(next_url) Page.new(parsed_response, type) end end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb index 7b639492d38..94adaacc9b5 100644 --- a/lib/bitbucket/representation/base.rb +++ b/lib/bitbucket/representation/base.rb @@ -5,6 +5,10 @@ module Bitbucket @raw = raw end + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + private attr_reader :raw diff --git a/lib/bitbucket/representation/url.rb b/lib/bitbucket/representation/url.rb deleted file mode 100644 index 24ae1048013..00000000000 --- a/lib/bitbucket/representation/url.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Bitbucket - module Representation - class Url < Representation::Base - def to_s - raw.dig('links', 'self', 'href') - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 0f583b07e93..825d43e6589 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -50,6 +50,13 @@ module Gitlab if issue.persisted? client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Chenged title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + note = @formatter.author_line(comment.author) note += comment.note -- cgit v1.2.1 From 98c0eb0f75692b1281adda9bfb75e1fcc12cec6d Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 7 Dec 2016 15:54:32 +0200 Subject: BitBucket refactoring. Iteration 3 --- lib/bitbucket/client.rb | 26 ++++++++++---------------- lib/bitbucket/collection.rb | 2 +- lib/bitbucket/paginator.rb | 2 +- lib/gitlab/bitbucket_import/importer.rb | 6 +++--- 4 files changed, 15 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 3457c2c6454..e23da4556aa 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -6,35 +6,26 @@ module Bitbucket def issues(repo) path = "/repositories/#{repo}/issues" - paginator = Paginator.new(connection, path, :issue) - - Collection.new(paginator) + get_collection(path, :issue) end def issue_comments(repo, issue_id) path = "/repositories/#{repo}/issues/#{issue_id}/comments" - paginator = Paginator.new(connection, path, :comment) - - Collection.new(paginator) + get_collection(path, :comment) end def pull_requests(repo) path = "/repositories/#{repo}/pullrequests?state=ALL" - paginator = Paginator.new(connection, path, :pull_request) - - Collection.new(paginator) + get_collection(path, :pull_request) end def pull_request_comments(repo, pull_request) path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" - paginator = Paginator.new(connection, path, :pull_request_comment) - - Collection.new(paginator) + get_collection(path, :pull_request_comment) end def pull_request_diff(repo, pull_request) path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" - connection.get(path) end @@ -45,9 +36,7 @@ module Bitbucket def repos path = "/repositories/#{user.username}" - paginator = Paginator.new(connection, path, :repo) - - Collection.new(paginator) + get_collection(path, :repo) end def user @@ -60,5 +49,10 @@ module Bitbucket private attr_reader :connection + + def get_collection(path, type) + paginator = Paginator.new(connection, path, type) + Collection.new(paginator) + end end end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb index 9cc8947417c..3a9379ff680 100644 --- a/lib/bitbucket/collection.rb +++ b/lib/bitbucket/collection.rb @@ -3,7 +3,7 @@ module Bitbucket def initialize(paginator) super() do |yielder| loop do - paginator.next.each { |item| yielder << item } + paginator.items.each { |item| yielder << item } end end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb index 37f12328447..641a6ed79d6 100644 --- a/lib/bitbucket/paginator.rb +++ b/lib/bitbucket/paginator.rb @@ -11,7 +11,7 @@ module Bitbucket connection.set_default_query_parameters(pagelen: PAGE_LENGTH, sort: :created_on) end - def next + def items raise StopIteration unless has_next_page? @page = fetch_next_page diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 825d43e6589..fba382e6fea 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -50,7 +50,7 @@ module Gitlab if issue.persisted? client.issue_comments(repo, issue.iid).each do |comment| - # The note can be blank for issue service messages like "Chenged title: ..." + # The note can be blank for issue service messages like "Changed title: ..." # We would like to import those comments as well but there is no any # specific parameter that would allow to process them, it's just an empty comment. # To prevent our importer from just crashing or from creating useless empty comments @@ -70,8 +70,8 @@ module Gitlab end end end - rescue ActiveRecord::RecordInvalid - nil + rescue ActiveRecord::RecordInvalid => e + Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Couldn't import record properly #{e.message}") end def import_pull_requests -- cgit v1.2.1 From bd3bd9bcea11244c56a0f7b63a6afa6fe439bf01 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 7 Dec 2016 16:16:18 +0200 Subject: Remove outdated bitbucket_import.rb --- lib/gitlab/bitbucket_import.rb | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 lib/gitlab/bitbucket_import.rb (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb deleted file mode 100644 index 7298152e7e9..00000000000 --- a/lib/gitlab/bitbucket_import.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module BitbucketImport - mattr_accessor :public_key - @public_key = nil - end -end -- cgit v1.2.1 From 00cd864237d6c7ec57ecb49d304ca8dfa9e41d31 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 7 Dec 2016 18:04:02 +0200 Subject: BitBucket importer: import issues with labels --- lib/bitbucket/representation/issue.rb | 4 ++++ lib/gitlab/bitbucket_import/importer.rb | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) (limited to 'lib') diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index dc034c19750..6c8e9a4c244 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -7,6 +7,10 @@ module Bitbucket raw['id'] end + def kind + raw['kind'] + end + def author raw.dig('reporter', 'username') || 'Anonymous' end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index fba382e6fea..8852f5b0f3f 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,12 +1,18 @@ module Gitlab module BitbucketImport class Importer + LABELS = [{ title: 'bug', color: '#FF0000'}, + { title: 'enhancement', color: '#428BCA'}, + { title: 'proposal', color: '#69D100'}, + { title: 'task', color: '#7F8C8D'}].freeze + attr_reader :project, :client def initialize(project) @project = project @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @labels = {} end def execute @@ -34,10 +40,14 @@ module Gitlab def import_issues return unless repo.issues_enabled? + create_labels + client.issues(repo).each do |issue| description = @formatter.author_line(issue.author) description += issue.description + label_name = issue.kind + issue = project.issues.create( iid: issue.iid, title: issue.title, @@ -48,6 +58,8 @@ module Gitlab updated_at: issue.updated_at ) + assign_label(issue, label_name) + if issue.persisted? client.issue_comments(repo, issue.iid).each do |comment| # The note can be blank for issue service messages like "Changed title: ..." @@ -74,6 +86,16 @@ module Gitlab Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Couldn't import record properly #{e.message}") end + def create_labels + LABELS.each do |label| + @labels[label[:title]] = project.labels.create!(label) + end + end + + def assign_label(issue, label_name) + issue.labels << @labels[label_name] + end + def import_pull_requests pull_requests = client.pull_requests(repo) -- cgit v1.2.1 From 9ab1fe5e6536e311142d1daddd0d7c8e29eec20a Mon Sep 17 00:00:00 2001 From: Munken Date: Thu, 8 Dec 2016 21:52:57 +0000 Subject: Working inline math filter --- lib/banzai/filter/inline_math_filter.rb | 29 +++++++++++++++++++++++++++++ lib/banzai/pipeline/gfm_pipeline.rb | 1 + 2 files changed, 30 insertions(+) create mode 100644 lib/banzai/filter/inline_math_filter.rb (limited to 'lib') diff --git a/lib/banzai/filter/inline_math_filter.rb b/lib/banzai/filter/inline_math_filter.rb new file mode 100644 index 00000000000..a63116c12ce --- /dev/null +++ b/lib/banzai/filter/inline_math_filter.rb @@ -0,0 +1,29 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that adds class="code math" and removes the dolar sign in $`2+2`$. + # + class InlineMathFilter < HTML::Pipeline::Filter + def call + doc.xpath("descendant-or-self::text()[substring(., string-length(.)) = '$']"\ + "/following-sibling::*[name() = 'code']"\ + "/following-sibling::text()[starts-with(.,'$')]").each do |el| + closing = el + code = el.previous + code[:class] = 'code math' + opening = code.previous + + closing.content = closing.content[1..-1] + opening.content = opening.content[0..-2] + + closing + end + + puts doc + + doc + end + end + end +end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5da2d0b008c..2c81cbe56b3 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -6,6 +6,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::SanitizationFilter, + Filter::InlineMathFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLinkFilter, -- cgit v1.2.1 From 525c2a782e03afafdf9cf1948ab75e73092704fa Mon Sep 17 00:00:00 2001 From: Munken Date: Thu, 8 Dec 2016 22:39:37 +0000 Subject: Math works for inline syntax --- lib/banzai/filter/inline_math_filter.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/inline_math_filter.rb b/lib/banzai/filter/inline_math_filter.rb index a63116c12ce..1bbe602237a 100644 --- a/lib/banzai/filter/inline_math_filter.rb +++ b/lib/banzai/filter/inline_math_filter.rb @@ -20,8 +20,6 @@ module Banzai closing end - puts doc - doc end end -- cgit v1.2.1 From aa2c437fe05ea272e180527a848455599f9da916 Mon Sep 17 00:00:00 2001 From: Munken Date: Thu, 8 Dec 2016 22:59:37 +0000 Subject: Hacked in Math Lexer --- lib/banzai/filter/syntax_highlight_filter.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'lib') diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 026b81ac175..f6a813ac923 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,5 +1,29 @@ require 'rouge/plugins/redcarpet' +module Rouge + module Lexers + class Math < Lexer + title "Plain Text" + desc "A boring lexer that doesn't highlight anything" + + tag 'math' + aliases 'text' + filenames '*.txt' + mimetypes 'text/plain' + + default_options :token => 'Text' + + def token + @token ||= Token[option :token] + end + + def stream_tokens(string, &b) + yield self.token, string + end + end + end +end + module Banzai module Filter # HTML Filter to highlight fenced code blocks @@ -48,6 +72,9 @@ module Banzai end def lexer_for(language) + if language == 'math' + return Rouge::Lexers::Math.new + end (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new end -- cgit v1.2.1 From f7b09848c558b1a44571f27a9534c02c0501f181 Mon Sep 17 00:00:00 2001 From: Munken Date: Thu, 8 Dec 2016 23:09:34 +0000 Subject: Removed alias and filenames --- lib/banzai/filter/syntax_highlight_filter.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index f6a813ac923..278d4d92d08 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -7,8 +7,6 @@ module Rouge desc "A boring lexer that doesn't highlight anything" tag 'math' - aliases 'text' - filenames '*.txt' mimetypes 'text/plain' default_options :token => 'Text' -- cgit v1.2.1 From a36ee0ad4400b7e38bfa33ae99838dcb9527e870 Mon Sep 17 00:00:00 2001 From: Munken Date: Thu, 8 Dec 2016 23:13:44 +0000 Subject: Better location for math lexer --- lib/banzai/filter/syntax_highlight_filter.rb | 22 ---------------------- lib/rouge/lexers/math.rb | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 22 deletions(-) create mode 100644 lib/rouge/lexers/math.rb (limited to 'lib') diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 278d4d92d08..e7f6b715ba8 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,27 +1,5 @@ require 'rouge/plugins/redcarpet' -module Rouge - module Lexers - class Math < Lexer - title "Plain Text" - desc "A boring lexer that doesn't highlight anything" - - tag 'math' - mimetypes 'text/plain' - - default_options :token => 'Text' - - def token - @token ||= Token[option :token] - end - - def stream_tokens(string, &b) - yield self.token, string - end - end - end -end - module Banzai module Filter # HTML Filter to highlight fenced code blocks diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb new file mode 100644 index 00000000000..ae980da8283 --- /dev/null +++ b/lib/rouge/lexers/math.rb @@ -0,0 +1,21 @@ +module Rouge + module Lexers + class Math < Lexer + title "Plain Text" + desc "A boring lexer that doesn't highlight anything" + + tag 'math' + mimetypes 'text/plain' + + default_options :token => 'Text' + + def token + @token ||= Token[option :token] + end + + def stream_tokens(string, &b) + yield self.token, string + end + end + end +end \ No newline at end of file -- cgit v1.2.1 From cc30a9f7ed436fd906c1e24a195414f2f84ee61c Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Fri, 9 Dec 2016 17:25:42 +0200 Subject: Fix rubocop[ci skip] --- lib/gitlab/bitbucket_import/importer.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 8852f5b0f3f..e00a90da980 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,10 +1,10 @@ module Gitlab module BitbucketImport class Importer - LABELS = [{ title: 'bug', color: '#FF0000'}, - { title: 'enhancement', color: '#428BCA'}, - { title: 'proposal', color: '#69D100'}, - { title: 'task', color: '#7F8C8D'}].freeze + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze attr_reader :project, :client -- cgit v1.2.1 From ff2193a3db558214fab90bb644be6967a03176a0 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Fri, 9 Dec 2016 19:40:22 +0200 Subject: Fix specs --- lib/gitlab/bitbucket_import/importer.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index e00a90da980..a0a17333185 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -58,7 +58,7 @@ module Gitlab updated_at: issue.updated_at ) - assign_label(issue, label_name) + issue.labels << @labels[label_name] if issue.persisted? client.issue_comments(repo, issue.iid).each do |comment| @@ -92,10 +92,6 @@ module Gitlab end end - def assign_label(issue, label_name) - issue.labels << @labels[label_name] - end - def import_pull_requests pull_requests = client.pull_requests(repo) -- cgit v1.2.1 From 091970208e0c8e7aefb6e7dcfafb4c81188c27cf Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 9 Dec 2016 15:17:13 -0800 Subject: Return repositories to which user is a member, not just owner --- lib/bitbucket/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index e23da4556aa..a9f405e659b 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -35,7 +35,7 @@ module Bitbucket end def repos - path = "/repositories/#{user.username}" + path = "/repositories/#{user.username}?role=member" get_collection(path, :repo) end -- cgit v1.2.1 From 1d7f85aeef624a83f0b225217a23c8f5189cde54 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 9 Dec 2016 15:28:49 -0800 Subject: Fix query for importing all projects for member --- lib/bitbucket/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index a9f405e659b..5c2ef2a4509 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -35,7 +35,7 @@ module Bitbucket end def repos - path = "/repositories/#{user.username}?role=member" + path = "/repositories?role=member" get_collection(path, :repo) end -- cgit v1.2.1 From 314c4746bc24a31efe88b142cd83ab36c3473cc9 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 12 Dec 2016 16:16:51 +0200 Subject: Specs for Bitbucket::Connections and Bitbucket::Collections --- lib/bitbucket/connection.rb | 20 ++++++++------------ lib/bitbucket/paginator.rb | 4 +--- 2 files changed, 9 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index 692a596c057..c150a20761e 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -15,18 +15,6 @@ module Bitbucket @refresh_token = options[:refresh_token] end - def client - @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) - end - - def connection - @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) - end - - def set_default_query_parameters(params = {}) - @default_query.merge!(params) - end - def get(path, extra_query = {}) refresh! if expired? @@ -52,6 +40,14 @@ module Bitbucket attr_reader :expires_at, :expires_in, :refresh_token, :token + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + def build_url(path) return path if path.starts_with?(root_url) diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb index 641a6ed79d6..b38cd99855c 100644 --- a/lib/bitbucket/paginator.rb +++ b/lib/bitbucket/paginator.rb @@ -7,8 +7,6 @@ module Bitbucket @type = type @url = url @page = nil - - connection.set_default_query_parameters(pagelen: PAGE_LENGTH, sort: :created_on) end def items @@ -31,7 +29,7 @@ module Bitbucket end def fetch_next_page - parsed_response = connection.get(next_url) + parsed_response = connection.get(next_url, { pagelen: PAGE_LENGTH, sort: :created_on }) Page.new(parsed_response, type) end end -- cgit v1.2.1 From 3a0fecb4924f1a6dbcc3e61041e0cac95ec3b21b Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 12 Dec 2016 17:29:25 +0200 Subject: Spec for bitbucket page --- lib/bitbucket/page.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb index 8f50f67f84d..2b0a3fe7b1a 100644 --- a/lib/bitbucket/page.rb +++ b/lib/bitbucket/page.rb @@ -23,6 +23,7 @@ module Bitbucket def parse_values(raw, bitbucket_rep_class) return [] unless raw['values'] && raw['values'].is_a?(Array) + bitbucket_rep_class.decorate(raw['values']) end -- cgit v1.2.1 From a2be395401f6320d2722bbd98de0c046d05f0480 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 12 Dec 2016 20:41:55 +0200 Subject: specs for bitbucket representers --- lib/bitbucket/representation/pull_request_comment.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index ae2b069d6a2..4f3809fbcea 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -6,15 +6,15 @@ module Bitbucket end def file_path - inline.fetch('path', nil) + inline.fetch('path') end def old_pos - inline.fetch('from', nil) + inline.fetch('from') end def new_pos - inline.fetch('to', nil) + inline.fetch('to') end def parent_id -- cgit v1.2.1 From 0057ed1e69bc203d82fd3e8dfa6db7ea6a9b1de7 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 13 Dec 2016 17:59:21 +0200 Subject: BB importer: Fixed after code review[ci skip] --- lib/bitbucket/paginator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb index b38cd99855c..135d0d55674 100644 --- a/lib/bitbucket/paginator.rb +++ b/lib/bitbucket/paginator.rb @@ -29,7 +29,7 @@ module Bitbucket end def fetch_next_page - parsed_response = connection.get(next_url, { pagelen: PAGE_LENGTH, sort: :created_on }) + parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on) Page.new(parsed_response, type) end end -- cgit v1.2.1 From e39f024029b46322c1bf24409fd5ce7bfcef2da5 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Tue, 13 Dec 2016 21:28:04 +0200 Subject: BB importer: Adding created_by only when used is not found[ci skip] --- lib/bitbucket/representation/base.rb | 4 ++++ lib/bitbucket/representation/comment.rb | 2 +- lib/bitbucket/representation/issue.rb | 2 +- lib/bitbucket/representation/pull_request.rb | 2 +- lib/bitbucket/representation/user.rb | 6 +++++- lib/gitlab/bitbucket_import/importer.rb | 23 +++++++++++++++++------ 6 files changed, 29 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb index 94adaacc9b5..fd622d333da 100644 --- a/lib/bitbucket/representation/base.rb +++ b/lib/bitbucket/representation/base.rb @@ -5,6 +5,10 @@ module Bitbucket @raw = raw end + def user_representation(raw) + User.new(raw) + end + def self.decorate(entries) entries.map { |entry| new(entry)} end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb index 94bc18cbfab..bc40f891cd3 100644 --- a/lib/bitbucket/representation/comment.rb +++ b/lib/bitbucket/representation/comment.rb @@ -2,7 +2,7 @@ module Bitbucket module Representation class Comment < Representation::Base def author - user.fetch('username', 'Anonymous') + user_representation(user) end def note diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index 6c8e9a4c244..90adfa3331a 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -12,7 +12,7 @@ module Bitbucket end def author - raw.dig('reporter', 'username') || 'Anonymous' + user_representation(raw.fetch('reporter', {})) end def description diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index e7b1f99e9a6..96992003d24 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -2,7 +2,7 @@ module Bitbucket module Representation class PullRequest < Representation::Base def author - raw.fetch('author', {}).fetch('username', 'Anonymous') + user_representation(raw.fetch('author', {})) end def description diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb index ba6b7667b49..6025a9f0653 100644 --- a/lib/bitbucket/representation/user.rb +++ b/lib/bitbucket/representation/user.rb @@ -2,7 +2,11 @@ module Bitbucket module Representation class User < Representation::Base def username - raw['username'] + raw['username'] || 'Anonymous' + end + + def uuid + raw['uuid'] end end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index a0a17333185..519a109c0c8 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -24,15 +24,23 @@ module Gitlab private - def gitlab_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + def gitlab_user_id(project, user) + if user.uuid + user = find_user_by_uuid(user.uuid) (user && user.id) || project.creator_id else project.creator_id end end + def find_user_by_uuid(uuid) + User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", uuid) + end + + def existing_gitlab_user?(user) + user.uuid && find_user_by_uuid(user.uuid) + end + def repo @repo ||= client.repo(project.import_source) end @@ -43,7 +51,8 @@ module Gitlab create_labels client.issues(repo).each do |issue| - description = @formatter.author_line(issue.author) + description = '' + description += @formatter.author_line(issue.author.username) unless existing_gitlab_user?(issue.author) description += issue.description label_name = issue.kind @@ -69,7 +78,8 @@ module Gitlab # we do this check. next unless comment.note.present? - note = @formatter.author_line(comment.author) + note = '' + note += @formatter.author_line(comment.author.username) unless existing_gitlab_user?(comment.author) note += comment.note issue.notes.create!( @@ -97,7 +107,8 @@ module Gitlab pull_requests.each do |pull_request| begin - description = @formatter.author_line(pull_request.author) + description = '' + description += @formatter.author_line(pull_request.author.username) unless existing_gitlab_user?(pull_request.author) description += pull_request.description merge_request = project.merge_requests.create( -- cgit v1.2.1 From f20ea1f5cb354db0afe18e4021e1d2fb439c2e06 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 14 Dec 2016 11:53:29 +0200 Subject: Fix BB authentication[ci skip] --- lib/omniauth/strategies/bitbucket.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb index 475aad5970f..5a7d67c2390 100644 --- a/lib/omniauth/strategies/bitbucket.rb +++ b/lib/omniauth/strategies/bitbucket.rb @@ -11,10 +11,6 @@ module OmniAuth token_url: 'https://bitbucket.org/site/oauth2/access_token' } - def callback_url - full_host + script_name + callback_path - end - uid do raw_info['username'] end @@ -28,7 +24,7 @@ module OmniAuth end def raw_info - @raw_info ||= access_token.get('user').parsed + @raw_info ||= access_token.get('api/2.0/user').parsed end def primary_email @@ -37,7 +33,7 @@ module OmniAuth end def emails - email_response = access_token.get('user/emails').parsed + email_response = access_token.get('api/2.0/user/emails').parsed @emails ||= email_response && email_response['values'] || nil end end -- cgit v1.2.1 From 8f0cef0b6e5e950efdf3ebfe8f9f846095fff9d9 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 14 Dec 2016 12:35:10 +0200 Subject: BB importer: Refactoring user importing logic[ci skip] --- lib/bitbucket/representation/base.rb | 4 ---- lib/bitbucket/representation/comment.rb | 2 +- lib/bitbucket/representation/issue.rb | 2 +- lib/bitbucket/representation/pull_request.rb | 2 +- lib/bitbucket/representation/user.rb | 6 +----- lib/gitlab/bitbucket_import/importer.rb | 20 ++++++++++---------- 6 files changed, 14 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb index fd622d333da..94adaacc9b5 100644 --- a/lib/bitbucket/representation/base.rb +++ b/lib/bitbucket/representation/base.rb @@ -5,10 +5,6 @@ module Bitbucket @raw = raw end - def user_representation(raw) - User.new(raw) - end - def self.decorate(entries) entries.map { |entry| new(entry)} end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb index bc40f891cd3..3c75e9368fa 100644 --- a/lib/bitbucket/representation/comment.rb +++ b/lib/bitbucket/representation/comment.rb @@ -2,7 +2,7 @@ module Bitbucket module Representation class Comment < Representation::Base def author - user_representation(user) + user['username'] end def note diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index 90adfa3331a..ffe8a65d839 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -12,7 +12,7 @@ module Bitbucket end def author - user_representation(raw.fetch('reporter', {})) + raw.dig('reporter', 'username') end def description diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index 96992003d24..e37c9a62c0e 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -2,7 +2,7 @@ module Bitbucket module Representation class PullRequest < Representation::Base def author - user_representation(raw.fetch('author', {})) + raw.dig('author', 'username') end def description diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb index 6025a9f0653..ba6b7667b49 100644 --- a/lib/bitbucket/representation/user.rb +++ b/lib/bitbucket/representation/user.rb @@ -2,11 +2,7 @@ module Bitbucket module Representation class User < Representation::Base def username - raw['username'] || 'Anonymous' - end - - def uuid - raw['uuid'] + raw['username'] end end end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 519a109c0c8..b6a0b122cdb 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -24,21 +24,21 @@ module Gitlab private - def gitlab_user_id(project, user) - if user.uuid - user = find_user_by_uuid(user.uuid) + def gitlab_user_id(project, username) + if username + user = find_user(username) (user && user.id) || project.creator_id else project.creator_id end end - def find_user_by_uuid(uuid) - User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", uuid) + def find_user(username) + User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) end - def existing_gitlab_user?(user) - user.uuid && find_user_by_uuid(user.uuid) + def existing_gitlab_user?(username) + username && find_user(username) end def repo @@ -52,7 +52,7 @@ module Gitlab client.issues(repo).each do |issue| description = '' - description += @formatter.author_line(issue.author.username) unless existing_gitlab_user?(issue.author) + description += @formatter.author_line(issue.author) unless existing_gitlab_user?(issue.author) description += issue.description label_name = issue.kind @@ -79,7 +79,7 @@ module Gitlab next unless comment.note.present? note = '' - note += @formatter.author_line(comment.author.username) unless existing_gitlab_user?(comment.author) + note += @formatter.author_line(comment.author) unless existing_gitlab_user?(comment.author) note += comment.note issue.notes.create!( @@ -108,7 +108,7 @@ module Gitlab pull_requests.each do |pull_request| begin description = '' - description += @formatter.author_line(pull_request.author.username) unless existing_gitlab_user?(pull_request.author) + description += @formatter.author_line(pull_request.author) unless existing_gitlab_user?(pull_request.author) description += pull_request.description merge_request = project.merge_requests.create( -- cgit v1.2.1 From 6bbe2f118ee17ac8b1d43a77f4020c048c427b77 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Wed, 14 Dec 2016 15:18:30 +0200 Subject: BB importer: More advanced error handling --- lib/gitlab/bitbucket_import/importer.rb | 80 +++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index b6a0b122cdb..567f2b314aa 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -6,24 +6,34 @@ module Gitlab { title: 'proposal', color: '#69D100' }, { title: 'task', color: '#7F8C8D' }].freeze - attr_reader :project, :client + attr_reader :project, :client, :errors def initialize(project) @project = project @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new @labels = {} + @errors = [] end def execute import_issues import_pull_requests + handle_errors true end private + def handle_errors + return unless errors.any? + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + def gitlab_user_id(project, username) if username user = find_user(username) @@ -51,21 +61,25 @@ module Gitlab create_labels client.issues(repo).each do |issue| - description = '' - description += @formatter.author_line(issue.author) unless existing_gitlab_user?(issue.author) - description += issue.description - - label_name = issue.kind - - issue = project.issues.create( - iid: issue.iid, - title: issue.title, - description: description, - state: issue.state, - author_id: gitlab_user_id(project, issue.author), - created_at: issue.created_at, - updated_at: issue.updated_at - ) + begin + description = '' + description += @formatter.author_line(issue.author) unless existing_gitlab_user?(issue.author) + description += issue.description + + label_name = issue.kind + + issue = project.issues.create!( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gitlab_user_id(project, issue.author), + created_at: issue.created_at, + updated_at: issue.updated_at + ) + rescue StandardError => e + errors << { type: :issue, iid: issue.iid, errors: e.message } + end issue.labels << @labels[label_name] @@ -82,18 +96,20 @@ module Gitlab note += @formatter.author_line(comment.author) unless existing_gitlab_user?(comment.author) note += comment.note - issue.notes.create!( - project: project, - note: note, - author_id: gitlab_user_id(project, comment.author), - created_at: comment.created_at, - updated_at: comment.updated_at - ) + begin + issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + rescue StandardError => e + errors << { type: :issue_comment, iid: issue.iid, errors: e.message } + end end end end - rescue ActiveRecord::RecordInvalid => e - Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Couldn't import record properly #{e.message}") end def create_labels @@ -129,8 +145,8 @@ module Gitlab ) import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? - rescue ActiveRecord::RecordInvalid - Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request #{e.message}") + rescue StandardError => e + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } end end end @@ -169,9 +185,8 @@ module Gitlab type: 'DiffNote') merge_request.notes.create!(attributes) - rescue ActiveRecord::RecordInvalid => e - Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request comment #{e.message}") - nil + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } end end end @@ -192,9 +207,8 @@ module Gitlab pr_comments.each do |comment| begin merge_request.notes.create!(pull_request_comment_attributes(comment)) - rescue ActiveRecord::RecordInvalid => e - Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid standalone pull request comment #{e.message}") - nil + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } end end end -- cgit v1.2.1 From 2d170a20dc4cd3423ac7994c797cae8fbed263ba Mon Sep 17 00:00:00 2001 From: Munken Date: Fri, 9 Dec 2016 00:15:08 +0000 Subject: Render math in Asciidoc and Markdown with KaTeX using code blocks --- lib/banzai/filter/inline_math_filter.rb | 27 --------------- lib/banzai/filter/math_filter.rb | 51 ++++++++++++++++++++++++++++ lib/banzai/filter/syntax_highlight_filter.rb | 3 -- lib/banzai/pipeline/gfm_pipeline.rb | 2 +- lib/gitlab/asciidoc.rb | 31 ++++++++++++++++- lib/gitlab/gon_helper.rb | 2 ++ lib/rouge/lexers/math.rb | 6 ++-- 7 files changed, 87 insertions(+), 35 deletions(-) delete mode 100644 lib/banzai/filter/inline_math_filter.rb create mode 100644 lib/banzai/filter/math_filter.rb (limited to 'lib') diff --git a/lib/banzai/filter/inline_math_filter.rb b/lib/banzai/filter/inline_math_filter.rb deleted file mode 100644 index 1bbe602237a..00000000000 --- a/lib/banzai/filter/inline_math_filter.rb +++ /dev/null @@ -1,27 +0,0 @@ -require 'uri' - -module Banzai - module Filter - # HTML filter that adds class="code math" and removes the dolar sign in $`2+2`$. - # - class InlineMathFilter < HTML::Pipeline::Filter - def call - doc.xpath("descendant-or-self::text()[substring(., string-length(.)) = '$']"\ - "/following-sibling::*[name() = 'code']"\ - "/following-sibling::text()[starts-with(.,'$')]").each do |el| - closing = el - code = el.previous - code[:class] = 'code math' - opening = code.previous - - closing.content = closing.content[1..-1] - opening.content = opening.content[0..-2] - - closing - end - - doc - end - end - end -end diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb new file mode 100644 index 00000000000..cb037f89337 --- /dev/null +++ b/lib/banzai/filter/math_filter.rb @@ -0,0 +1,51 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. + # + class MathFilter < HTML::Pipeline::Filter + # This picks out .... + INLINE_MATH = 'descendant-or-self::code'.freeze + + # Pick out a code block which is declared math + DISPLAY_MATH = "descendant-or-self::pre[contains(@class, 'math') and contains(@class, 'code')]".freeze + + # Attribute indicating inline or display math. + STYLE_ATTRIBUTE = 'data-math-style'.freeze + + # Class used for tagging elements that should be rendered + TAG_CLASS = 'js-render-math'.freeze + + INLINE_CLASSES = "code math #{TAG_CLASS}".freeze + + DOLLAR_SIGN = '$'.freeze + + def call + doc.xpath(INLINE_MATH).each do |code| + closing = code.next + opening = code.previous + + # We need a sibling before and after. + # They should end and start with $ respectively. + if closing && opening && + closing.content.first == DOLLAR_SIGN && + opening.content.last == DOLLAR_SIGN + + code[:class] = INLINE_CLASSES + code[STYLE_ATTRIBUTE] = 'inline' + closing.content = closing.content[1..-1] + opening.content = opening.content[0..-2] + end + end + + doc.xpath(DISPLAY_MATH).each do |el| + el[STYLE_ATTRIBUTE] = 'display' + el[:class] += " #{TAG_CLASS}" + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index e7f6b715ba8..026b81ac175 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -48,9 +48,6 @@ module Banzai end def lexer_for(language) - if language == 'math' - return Rouge::Lexers::Math.new - end (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 2c81cbe56b3..5a1f873496c 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -6,7 +6,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::SanitizationFilter, - Filter::InlineMathFilter, + Filter::MathFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLinkFilter, diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 9667df4ffb8..f77f412da56 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,4 +1,5 @@ require 'asciidoctor' +require 'asciidoctor/converter/html5' module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters @@ -23,7 +24,7 @@ module Gitlab def self.render(input, context, asciidoc_opts = {}) asciidoc_opts.reverse_merge!( safe: :secure, - backend: :html5, + backend: :gitlab_html5, attributes: [] ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) @@ -36,3 +37,31 @@ module Gitlab end end end + +module Gitlab + module Asciidoc + class Html5Converter < Asciidoctor::Converter::Html5Converter + extend Asciidoctor::Converter::Config + + register_for 'gitlab_html5' + + def stem(node) + return super unless node.style.to_sym == :latexmath + + %(#{node.content}) + end + + def inline_quoted(node) + return super unless node.type.to_sym == :latexmath + + %(#{node.text}) + end + + private + + def id_attribute(node) + node.id ? %( id="#{node.id}") : nil + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2c21804fe7a..4d4e04e9e35 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -8,6 +8,8 @@ module Gitlab gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.award_menu_url = emojis_path + gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') + gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') if current_user gon.current_user_id = current_user.id diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb index ae980da8283..80784adfd76 100644 --- a/lib/rouge/lexers/math.rb +++ b/lib/rouge/lexers/math.rb @@ -1,13 +1,13 @@ module Rouge module Lexers class Math < Lexer - title "Plain Text" + title "A passthrough lexer used for LaTeX input" desc "A boring lexer that doesn't highlight anything" tag 'math' mimetypes 'text/plain' - default_options :token => 'Text' + default_options token: 'Text' def token @token ||= Token[option :token] @@ -18,4 +18,4 @@ module Rouge end end end -end \ No newline at end of file +end -- cgit v1.2.1 From f463ef5ec58fd26ad5368f57a29d758756dafc3f Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Thu, 15 Dec 2016 18:12:33 +0800 Subject: Also use latest_status, feedback: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7333#note_20058857 --- lib/gitlab/badge/build/status.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb index f78dfd5b83a..b762d85b6e5 100644 --- a/lib/gitlab/badge/build/status.rb +++ b/lib/gitlab/badge/build/status.rb @@ -21,8 +21,7 @@ module Gitlab def status @project.pipelines .where(sha: @sha) - .latest(@ref) - .status || 'unknown' + .latest_status(@ref) || 'unknown' end def metadata -- cgit v1.2.1 From 6731ab5d76c34462f0b4424ff03c9646ad916b76 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 18 Aug 2016 16:31:44 +0200 Subject: Add Gitlab::Middleware::Multipart --- lib/gitlab/gfm/uploads_rewriter.rb | 19 +++++++- lib/gitlab/middleware/multipart.rb | 99 ++++++++++++++++++++++++++++++++++++++ lib/gitlab/workhorse.rb | 6 ++- 3 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 lib/gitlab/middleware/multipart.rb (limited to 'lib') diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index abc8c8c55e6..8fab5489616 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -1,3 +1,5 @@ +require 'fileutils' + module Gitlab module Gfm ## @@ -22,7 +24,9 @@ module Gitlab return markdown unless file.try(:exists?) new_uploader = FileUploader.new(target_project) - new_uploader.store!(file) + with_link_in_tmp_dir(file.file) do |open_tmp_file| + new_uploader.store!(open_tmp_file) + end new_uploader.to_markdown end end @@ -46,6 +50,19 @@ module Gitlab uploader.retrieve_from_store!(file) uploader.file end + + # Because the uploaders use 'move_to_store' we must have a temporary + # file that is allowed to be (re)moved. + def with_link_in_tmp_dir(file) + dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file)) + # The filename matters to Carrierwave so we make sure to preserve it + tmp_file = File.join(dir, File.basename(file)) + File.link(file, tmp_file) + # Open the file to placate Carrierwave + File.open(tmp_file) { |open_file| yield open_file } + ensure + FileUtils.rm_rf(dir) + end end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb new file mode 100644 index 00000000000..65713e73a59 --- /dev/null +++ b/lib/gitlab/middleware/multipart.rb @@ -0,0 +1,99 @@ +# Gitlab::Middleware::Multipart - a Rack::Multipart replacement +# +# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby +# process time to copy files around. This alternative solution uses +# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a +# location where copying should not be needed. +# +# When gitlab-workhorse finds files in a multipart MIME body it sends +# a signed message via a request header. This message lists the names of +# the multipart entries that gitlab-workhorse filtered out of the +# multipart structure and saved to tempfiles. Workhorse adds new entries +# in the multipart structure with paths to the tempfiles. +# +# The job of this Rack middleware is to detect and decode the message +# from workhorse. If present, it walks the Rack 'params' hash for the +# current request, opens the respective tempfiles, and inserts the open +# Ruby File objects in the params hash where Rack::Multipart would have +# put them. The goal is that application code deeper down can keep +# working the way it did with Rack::Multipart without changes. +# +# CAVEAT: the code that modifies the params hash is a bit complex. It is +# conceivable that certain Rack params structures will not be modified +# correctly. We are not aware of such bugs at this time though. +# + +module Gitlab + module Middleware + class Multipart + RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' + + class Handler + def initialize(env, message) + @request = Rack::Request.new(env) + @rewritten_fields = message['rewritten_fields'] + @open_files = [] + end + + def with_open_files + @rewritten_fields.each do |field, tmp_path| + parsed_field = Rack::Utils.parse_nested_query(field) + raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 + + key, value = parsed_field.first + if value.nil? + value = File.open(tmp_path) + @open_files << value + else + value = decorate_params_value(value, @request.params[key], tmp_path) + end + @request.update_param(key, value) + end + + yield + ensure + @open_files.each(&:close) + end + + # This function calls itself recursively + def decorate_params_value(path_hash, value_hash, tmp_path) + unless path_hash.is_a?(Hash) && path_hash.count == 1 + raise "invalid path: #{path_hash.inspect}" + end + path_key, path_value = path_hash.first + + unless value_hash.is_a?(Hash) && value_hash[path_key] + raise "invalid value hash: #{value_hash.inspect}" + end + + case path_value + when nil + value_hash[path_key] = File.open(tmp_path) + @open_files << value_hash[path_key] + value_hash + when Hash + decorate_params_value(path_value, value_hash[path_key], tmp_path) + value_hash + else + raise "unexpected path value: #{path_value.inspect}" + end + end + end + + def initialize(app) + @app = app + end + + def call(env) + encoded_message = env.delete(RACK_ENV_KEY) + return @app.call(env) if encoded_message.blank? + + message = Gitlab::Workhorse.decode_jwt(encoded_message)[0] + + Handler.new(env, message).with_open_files do + @app.call(env) + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 594439a5d4b..aeb1a26e1ba 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -117,8 +117,12 @@ module Gitlab end def verify_api_request!(request_headers) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) + end + + def decode_jwt(encoded_message) JWT.decode( - request_headers[INTERNAL_API_REQUEST_HEADER], + encoded_message, secret, true, { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, -- cgit v1.2.1 From 26628fb91a89bbe4998633eec00d2bd76cfb95c0 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Thu, 15 Dec 2016 14:19:28 +0200 Subject: BB importer: Fixed bug with putting expired token to a project.clone_url --- lib/bitbucket/client.rb | 4 ++-- lib/bitbucket/connection.rb | 4 ++-- lib/gitlab/bitbucket_import/project_creator.rb | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index 5c2ef2a4509..f8ee7e0f9ae 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -1,5 +1,7 @@ module Bitbucket class Client + attr_reader :connection + def initialize(options = {}) @connection = Connection.new(options) end @@ -48,8 +50,6 @@ module Bitbucket private - attr_reader :connection - def get_collection(path, type) paginator = Paginator.new(connection, path, type) Collection.new(paginator) diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index c150a20761e..7e55cf4deab 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -4,6 +4,8 @@ module Bitbucket DEFAULT_BASE_URI = 'https://api.bitbucket.org/' DEFAULT_QUERY = {} + attr_reader :expires_at, :expires_in, :refresh_token, :token + def initialize(options = {}) @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) @@ -38,8 +40,6 @@ module Bitbucket private - attr_reader :expires_at, :expires_in, :refresh_token, :token - def client @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b34be272af3..eb03882ab26 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -21,7 +21,7 @@ module Gitlab visibility_level: repo.visibility_level, import_type: 'bitbucket', import_source: repo.full_name, - import_url: repo.clone_url(@session_data[:token]), + import_url: repo.clone_url(session_data[:token]), import_data: { credentials: session_data } ).execute end -- cgit v1.2.1 From 6342ca367a32e55ec6b5ef8b3e8f9e13387b6615 Mon Sep 17 00:00:00 2001 From: Pedro Moreira da Silva Date: Wed, 30 Nov 2016 16:04:32 +0000 Subject: =?UTF-8?q?Improve=20`issue=20create=20=E2=80=A6`=20slash=20comman?= =?UTF-8?q?d=20with=20user=20input=20keys=20to=20create=20a=20newline=20in?= =?UTF-8?q?=20chat=20clients.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/gitlab/chat_commands/issue_create.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_create.rb index 1dba85c1b51..e6e8ce85e98 100644 --- a/lib/gitlab/chat_commands/issue_create.rb +++ b/lib/gitlab/chat_commands/issue_create.rb @@ -8,7 +8,7 @@ module Gitlab end def self.help_message - 'issue new \n<description>' + 'issue create <title>` *`Shift`* + *`Enter`* `<description>' end def self.allowed?(project, user) -- cgit v1.2.1 From 70e72e8938b169f88e5b014500a9afeb6ff66a2b Mon Sep 17 00:00:00 2001 From: Pedro Moreira da Silva <pedro@gitlab.com> Date: Thu, 15 Dec 2016 13:11:43 +0000 Subject: Rename `issue create` slash command to `issue new` --- lib/gitlab/chat_commands/issue_create.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_create.rb index e6e8ce85e98..cefb6775db8 100644 --- a/lib/gitlab/chat_commands/issue_create.rb +++ b/lib/gitlab/chat_commands/issue_create.rb @@ -8,7 +8,7 @@ module Gitlab end def self.help_message - 'issue create <title>` *`Shift`* + *`Enter`* `<description>' + 'issue new <title> *`⇧ Shift`*+*`↵ Enter`* <description>' end def self.allowed?(project, user) -- cgit v1.2.1 From 93a03cd92f6418fbeaf126c30c161ab40d377e94 Mon Sep 17 00:00:00 2001 From: Nick Thomas <nick@gitlab.com> Date: Thu, 8 Dec 2016 01:09:18 +0000 Subject: Add an environment slug --- lib/api/entities.rb | 2 +- lib/api/environments.rb | 3 +++ lib/api/helpers/custom_validators.rb | 14 ++++++++++++++ lib/gitlab/regex.rb | 9 +++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 lib/api/helpers/custom_validators.rb (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 01c0f5072ba..dfbb3ab86dd 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -629,7 +629,7 @@ module API end class EnvironmentBasic < Grape::Entity - expose :id, :name, :external_url + expose :id, :name, :slug, :external_url end class Environment < EnvironmentBasic diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 80bbd9bb6e4..1a7e68f0528 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -1,6 +1,7 @@ module API # Environments RESTfull API endpoints class Environments < Grape::API + include ::API::Helpers::CustomValidators include PaginationParams before { authenticate! } @@ -29,6 +30,7 @@ module API params do requires :name, type: String, desc: 'The name of the environment to be created' optional :external_url, type: String, desc: 'URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end post ':id/environments' do authorize! :create_environment, user_project @@ -50,6 +52,7 @@ module API requires :environment_id, type: Integer, desc: 'The environment ID' optional :name, type: String, desc: 'The new environment name' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end put ':id/environments/:environment_id' do authorize! :update_environment, user_project diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb new file mode 100644 index 00000000000..0a8f3073a50 --- /dev/null +++ b/lib/api/helpers/custom_validators.rb @@ -0,0 +1,14 @@ +module API + module Helpers + module CustomValidators + class Absence < Grape::Validations::Base + def validate_param!(attr_name, params) + return if params.respond_to?(:key?) && !params.key?(attr_name) + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) + end + end + end + end +end + +Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 7c711d581e8..9e0b0e5ea98 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -131,5 +131,14 @@ module Gitlab def kubernetes_namespace_regex_message "can contain only letters, digits or '-', and cannot start or end with '-'" end + + def environment_slug_regex + @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze + end + + def environment_slug_regex_message + "can contain only lowercase letters, digits, and '-'. " \ + "Must start with a letter, and cannot end with '-'" + end end end -- cgit v1.2.1 From 12db4cc0e70d3e249f3bf9fde85e336839422319 Mon Sep 17 00:00:00 2001 From: Douwe Maan <douwe@gitlab.com> Date: Fri, 9 Dec 2016 01:56:31 +0000 Subject: Merge branch 'jej-note-search-uses-finder' into 'security' Fix missing Note access checks in by moving Note#search to updated NoteFinder Split from !2024 to partially solve https://gitlab.com/gitlab-org/gitlab-ce/issues/23867 ## Which fixes are in this MR? :warning: - Potentially untested :bomb: - No test coverage :traffic_light: - Test coverage of some sort exists (a test failed when error raised) :vertical_traffic_light: - Test coverage of return value (a test failed when nil used) :white_check_mark: - Permissions check tested ### Note lookup without access check - [x] :white_check_mark: app/finders/notes_finder.rb:13 :download_code check - [x] :white_check_mark: app/finders/notes_finder.rb:19 `SnippetsFinder` - [x] :white_check_mark: app/models/note.rb:121 [`Issue#visible_to_user`] - [x] :white_check_mark: lib/gitlab/project_search_results.rb:113 - This is the only use of `app/models/note.rb:121` above, but importantly has no access checks at all. This means it leaks MR comments and snippets when those features are `team-only` in addition to the issue comments which would be fixed by `app/models/note.rb:121`. - It is only called from SearchController where `can?(current_user, :download_code, @project)` is checked, so commit comments are not leaked. ### Previous discussions - [x] https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/2024/diffs#b915c5267a63628b0bafd23d37792ae73ceae272_13_13 `: download_code` check on commit - [x] https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/2024/diffs#b915c5267a63628b0bafd23d37792ae73ceae272_19_19 `SnippetsFinder` should be used - `SnippetsFinder` should check if the snippets feature is enabled -> https://gitlab.com/gitlab-org/gitlab-ce/issues/25223 ### Acceptance criteria met? - [x] Tests added for new code - [x] TODO comments removed - [x] Squashed and removed skipped tests - [x] Changelog entry - [ ] State Gitlab versions affected and issue severity in description - [ ] Create technical debt issue for NotesFinder. - Either split into `NotesFinder::ForTarget` and `NotesFinder::Search` or consider object per notable type such as `NotesFinder::OnIssue`. For the first option could create `NotesFinder::Base` which is either inherited from or which can be included in the other two. - Avoid case statement anti-pattern in this finder with use of `NotesFinder::OnCommit` etc. Consider something on the finder for this? `Model.finder(user, project)` - Move `inc_author` to the controller, and implement `related_notes` to replace `non_diff_notes`/`mr_and_commit_notes` See merge request !2035 --- lib/gitlab/project_search_results.rb | 2 +- lib/gitlab/sql/union.rb | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 66e6b29e798..6bdf3db9cb8 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -110,7 +110,7 @@ module Gitlab end def notes - @notes ||= project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') + @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') end def commits diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 1cd89b3a9c4..222021e8802 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -22,9 +22,7 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ActiveRecord::Base.connection.unprepared_statement do - @relations.map do |rel| - rel.reorder(nil).to_sql - end + @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end fragments.join("\nUNION\n") -- cgit v1.2.1 From a5ccaded656fb215f1f8d503b88c8f28bf90ce68 Mon Sep 17 00:00:00 2001 From: Felipe Artur <felipefac@gmail.com> Date: Tue, 6 Dec 2016 15:59:03 -0200 Subject: Change SlackService to SlackNotificationsService --- lib/api/services.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/services.rb b/lib/api/services.rb index b1e072b4f47..59232c84c24 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -473,7 +473,7 @@ module API desc: 'The description of the tracker' } ], - 'slack' => [ + 'slack-notification' => [ { required: true, name: :webhook, @@ -493,6 +493,14 @@ module API desc: 'The channel name' } ], + 'mattermost-notification' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + } + ], 'teamcity' => [ { required: true, -- cgit v1.2.1 From de0dfb10d47da609b3cca1f0e5310c14e5b1572c Mon Sep 17 00:00:00 2001 From: Munken <mm.munk@gmail.com> Date: Thu, 15 Dec 2016 18:03:57 +0000 Subject: Don't open Asciidoc module twice --- lib/gitlab/asciidoc.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index f77f412da56..fa234284361 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -35,11 +35,7 @@ module Gitlab html.html_safe end - end -end -module Gitlab - module Asciidoc class Html5Converter < Asciidoctor::Converter::Html5Converter extend Asciidoctor::Converter::Config -- cgit v1.2.1 From 7fa06ed55d18af4d055041eb27d38fecf9b5548f Mon Sep 17 00:00:00 2001 From: Timothy Andrew <mail@timothyandrew.net> Date: Tue, 22 Nov 2016 14:34:23 +0530 Subject: Calls to the API are checked for scope. - Move the `Oauth2::AccessTokenValidationService` class to `AccessTokenValidationService`, since it is now being used for personal access token validation as well. - Each API endpoint declares the scopes it accepts (if any). Currently, the top level API module declares the `api` scope, and the `Users` API module declares the `read_user` scope (for GET requests). - Move the `find_user_by_private_token` from the API `Helpers` module to the `APIGuard` module, to avoid littering `Helpers` with more auth-related methods to support `find_user_by_private_token` --- lib/api/api.rb | 2 ++ lib/api/api_guard.rb | 62 ++++++++++++++++++++++++++++++++++++++-------------- lib/api/helpers.rb | 15 +++---------- lib/api/users.rb | 5 ++++- lib/gitlab/auth.rb | 4 ++++ 5 files changed, 58 insertions(+), 30 deletions(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index cec2702e44d..9d5adffd8f4 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,8 @@ module API include APIGuard version 'v3', using: :path + before { allow_access_with_scope :api } + rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 8cc7a26f1fa..cd266669b1e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,6 +6,9 @@ module API module APIGuard extend ActiveSupport::Concern + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + PRIVATE_TOKEN_PARAM = :private_token + included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| @@ -41,30 +44,59 @@ module API # Defaults to empty array. # def doorkeeper_guard(scopes: []) - access_token = find_access_token - return nil unless access_token - - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + if access_token = find_access_token + case AccessTokenValidationService.validate(access_token, scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + when AccessTokenValidationService::EXPIRED + raise ExpiredError + when AccessTokenValidationService::REVOKED + raise RevokedError + when AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) + end + end + end - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + def find_user_by_private_token(scopes: []) + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + return nil unless token_string.present? - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) end def current_user @current_user end + # Set the authorization scope(s) allowed for the current request. + # + # Note: A call to this method adds to any previous scopes in place. This is done because + # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then + # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the + # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they + # need to be stored. + def allow_access_with_scope(*scopes) + @scopes ||= [] + @scopes.concat(scopes.map(&:to_s)) + end + private + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end + + def find_user_by_personal_access_token(token_string, scopes) + access_token = PersonalAccessToken.active.find_by_token(token_string) + return unless access_token + + if AccessTokenValidationService.sufficient_scope?(access_token, scopes) + User.find(access_token.user_id) + end + end + def find_access_token @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) end @@ -72,10 +104,6 @@ module API def doorkeeper_request @doorkeeper_request ||= ActionDispatch::Request.new(env) end - - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end end module ClassMethods diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 746849ef4c0..4be659fc20b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -2,8 +2,6 @@ module API module Helpers include Gitlab::Utils - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo @@ -308,7 +306,7 @@ module API private def private_token - params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] + params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER] end def warden @@ -323,18 +321,11 @@ module API warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) end - def find_user_by_private_token - token = private_token - return nil unless token.present? - - User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) - end - def initial_current_user return @initial_current_user if defined?(@initial_current_user) - @initial_current_user ||= find_user_by_private_token - @initial_current_user ||= doorkeeper_guard + @initial_current_user ||= find_user_by_private_token(scopes: @scopes) + @initial_current_user ||= doorkeeper_guard(scopes: @scopes) @initial_current_user ||= find_user_from_warden unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? diff --git a/lib/api/users.rb b/lib/api/users.rb index c7db2d71017..0842c3874c5 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -2,7 +2,10 @@ module API class Users < Grape::API include PaginationParams - before { authenticate! } + before do + allow_access_with_scope :read_user if request.get? + authenticate! + end resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do helpers do diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index aca5d0020cf..c3c464248ef 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,10 @@ module Gitlab module Auth class MissingPersonalTokenError < StandardError; end + SCOPES = [:api, :read_user] + DEFAULT_SCOPES = [:api] + OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES + class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? -- cgit v1.2.1 From 36b3210b9ec4fffd9fa5a73626907e8a6a59f435 Mon Sep 17 00:00:00 2001 From: Timothy Andrew <mail@timothyandrew.net> Date: Tue, 22 Nov 2016 14:43:37 +0530 Subject: Validate access token scopes in `Gitlab::Auth` - This module is used for git-over-http, as well as JWT. - The only valid scope here is `api`, currently. --- lib/gitlab/auth.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c3c464248ef..c6a23aa2bdf 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -92,7 +92,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - if token && token.accessible? + if token && token.accessible? && token_has_scope?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) end @@ -101,12 +101,20 @@ module Gitlab def personal_access_token_check(login, password) if login && password - user = User.find_by_personal_access_token(password) + token = PersonalAccessToken.active.find_by_token(password) validation = User.by_login(login) - Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + + if token && token.user == validation && token_has_scope?(token) + Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) + end + end end + def token_has_scope?(token) + AccessTokenValidationService.sufficient_scope?(token, ['api']) + end + def lfs_token_check(login, password) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) -- cgit v1.2.1 From 4d6da770de94f4bf140507cdf43461b67269ce28 Mon Sep 17 00:00:00 2001 From: Timothy Andrew <mail@timothyandrew.net> Date: Thu, 24 Nov 2016 13:07:22 +0530 Subject: Implement minor changes from @dbalexandre's review. - Mainly whitespace changes. - Require the migration adding the `scope` column to the `personal_access_tokens` table to have downtime, since API calls will fail if the new code is in place, but the migration hasn't run. - Minor refactoring - load `@scopes` in a `before_action`, since we're doing it in three different places. --- lib/api/api_guard.rb | 26 +++++++++++++++----------- lib/gitlab/auth.rb | 1 - 2 files changed, 15 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index cd266669b1e..563224a580f 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -44,17 +44,21 @@ module API # Defaults to empty array. # def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case AccessTokenValidationService.validate(access_token, scopes: scopes) - when AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when AccessTokenValidationService::EXPIRED - raise ExpiredError - when AccessTokenValidationService::REVOKED - raise RevokedError - when AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + access_token = find_access_token + return nil unless access_token + + case AccessTokenValidationService.validate(access_token, scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + + when AccessTokenValidationService::EXPIRED + raise ExpiredError + + when AccessTokenValidationService::REVOKED + raise RevokedError + + when AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c6a23aa2bdf..c425702fd75 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -107,7 +107,6 @@ module Gitlab if token && token.user == validation && token_has_scope?(token) Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) end - end end -- cgit v1.2.1 From dc95bcbb165289d9754e6bf66288c8d4350f6e57 Mon Sep 17 00:00:00 2001 From: Timothy Andrew <mail@timothyandrew.net> Date: Thu, 24 Nov 2016 14:39:12 +0530 Subject: Refactor access token validation in `Gitlab::Auth` - Based on @dbalexandre's review - Extract token validity conditions into two separate methods, for personal access tokens and OAuth tokens. --- lib/gitlab/auth.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c425702fd75..c21afaa1551 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -92,7 +92,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - if token && token.accessible? && token_has_scope?(token) + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) end @@ -104,12 +104,20 @@ module Gitlab token = PersonalAccessToken.active.find_by_token(password) validation = User.by_login(login) - if token && token.user == validation && token_has_scope?(token) + if valid_personal_access_token?(token, validation) Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) end end end + def valid_oauth_token?(token) + token && token.accessible? && token_has_scope?(token) + end + + def valid_personal_access_token?(token, user) + token && token.user == user && token_has_scope?(token) + end + def token_has_scope?(token) AccessTokenValidationService.sufficient_scope?(token, ['api']) end -- cgit v1.2.1 From b303948ff549ce57d3b6985c2c366dfcdc5a2ca3 Mon Sep 17 00:00:00 2001 From: Timothy Andrew <mail@timothyandrew.net> Date: Mon, 5 Dec 2016 22:55:53 +0530 Subject: Convert AccessTokenValidationService into a class. - Previously, AccessTokenValidationService was a module, and all its public methods accepted a token. It makes sense to convert it to a class which accepts a token during initialization. - Also rename the `sufficient_scope?` method to `include_any_scope?` - Based on feedback from @rymai --- lib/api/api_guard.rb | 4 ++-- lib/gitlab/auth.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 563224a580f..df6db140d0e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -47,7 +47,7 @@ module API access_token = find_access_token return nil unless access_token - case AccessTokenValidationService.validate(access_token, scopes: scopes) + case AccessTokenValidationService.new(access_token).validate(scopes: scopes) when AccessTokenValidationService::INSUFFICIENT_SCOPE raise InsufficientScopeError.new(scopes) @@ -96,7 +96,7 @@ module API access_token = PersonalAccessToken.active.find_by_token(token_string) return unless access_token - if AccessTokenValidationService.sufficient_scope?(access_token, scopes) + if AccessTokenValidationService.new(access_token).include_any_scope?(scopes) User.find(access_token.user_id) end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c21afaa1551..2879a4d2f5d 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -119,7 +119,7 @@ module Gitlab end def token_has_scope?(token) - AccessTokenValidationService.sufficient_scope?(token, ['api']) + AccessTokenValidationService.new(token).include_any_scope?(['api']) end def lfs_token_check(login, password) -- cgit v1.2.1 From 5becbe2495850923604c71b4c807666ea94819b3 Mon Sep 17 00:00:00 2001 From: Timothy Andrew <mail@timothyandrew.net> Date: Mon, 5 Dec 2016 22:58:19 +0530 Subject: Rename the `token_has_scope?` method. `valid_api_token?` is a better name. Scopes are just (potentially) one facet of a "valid" token. --- lib/gitlab/auth.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 2879a4d2f5d..8dda65c71ef 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -111,14 +111,14 @@ module Gitlab end def valid_oauth_token?(token) - token && token.accessible? && token_has_scope?(token) + token && token.accessible? && valid_api_token?(token) end def valid_personal_access_token?(token, user) - token && token.user == user && token_has_scope?(token) + token && token.user == user && valid_api_token?(token) end - def token_has_scope?(token) + def valid_api_token?(token) AccessTokenValidationService.new(token).include_any_scope?(['api']) end -- cgit v1.2.1 From 3b4e81eed50dac796de5720b9975125dc8de609b Mon Sep 17 00:00:00 2001 From: Valery Sizov <valery@gitlab.com> Date: Fri, 16 Dec 2016 12:12:53 +0200 Subject: BB importer: Milestone importer --- lib/bitbucket/representation/issue.rb | 4 ++++ lib/gitlab/bitbucket_import/importer.rb | 2 ++ 2 files changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index ffe8a65d839..3af731753d1 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -27,6 +27,10 @@ module Bitbucket raw['title'] end + def milestone + raw.dig('milestone', 'name') + end + def created_at raw['created_on'] end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 567f2b314aa..53c95ea4079 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -67,6 +67,7 @@ module Gitlab description += issue.description label_name = issue.kind + milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil issue = project.issues.create!( iid: issue.iid, @@ -74,6 +75,7 @@ module Gitlab description: description, state: issue.state, author_id: gitlab_user_id(project, issue.author), + milestone: milestone, created_at: issue.created_at, updated_at: issue.updated_at ) -- cgit v1.2.1 From b0501c34c478a528f2aa7633dfa6d13e9c61af64 Mon Sep 17 00:00:00 2001 From: Valery Sizov <valery@gitlab.com> Date: Fri, 16 Dec 2016 15:40:38 +0200 Subject: BB importer: address review comment --- lib/gitlab/bitbucket_import/importer.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 53c95ea4079..63a4407cb78 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -28,6 +28,7 @@ module Gitlab def handle_errors return unless errors.any? + project.update_column(:import_error, { message: 'The remote data could not be fully imported.', errors: errors @@ -35,15 +36,12 @@ module Gitlab end def gitlab_user_id(project, username) - if username - user = find_user(username) - (user && user.id) || project.creator_id - else - project.creator_id - end + user = find_user(username) + user.try(:id) || project.creator_id end def find_user(username) + return nil unless username User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) end -- cgit v1.2.1 From 7985b52286237b3801fc112b8bf3841599931c23 Mon Sep 17 00:00:00 2001 From: Valery Sizov <valery@gitlab.com> Date: Fri, 16 Dec 2016 17:35:31 +0200 Subject: BB importer: Adressed more review comments --- lib/gitlab/bitbucket_import/importer.rb | 34 ++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 63a4407cb78..f3760640655 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -6,7 +6,7 @@ module Gitlab { title: 'proposal', color: '#69D100' }, { title: 'task', color: '#7F8C8D' }].freeze - attr_reader :project, :client, :errors + attr_reader :project, :client, :errors, :users def initialize(project) @project = project @@ -14,6 +14,7 @@ module Gitlab @formatter = Gitlab::ImportFormatter.new @labels = {} @errors = [] + @users = {} end def execute @@ -36,17 +37,18 @@ module Gitlab end def gitlab_user_id(project, username) - user = find_user(username) - user.try(:id) || project.creator_id + find_user_id(username) || project.creator_id end - def find_user(username) + def find_user_id(username) return nil unless username - User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) - end - def existing_gitlab_user?(username) - username && find_user(username) + return users[username] if users.key?(username) + + users[username] = User.select(:id) + .joins(:identities) + .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) + .try(:id) end def repo @@ -58,16 +60,18 @@ module Gitlab create_labels + gitlab_issue = nil + client.issues(repo).each do |issue| begin description = '' - description += @formatter.author_line(issue.author) unless existing_gitlab_user?(issue.author) + description += @formatter.author_line(issue.author) unless find_user_id(issue.author) description += issue.description label_name = issue.kind milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil - issue = project.issues.create!( + gitlab_issue = project.issues.create!( iid: issue.iid, title: issue.title, description: description, @@ -81,9 +85,9 @@ module Gitlab errors << { type: :issue, iid: issue.iid, errors: e.message } end - issue.labels << @labels[label_name] + gitlab_issue.labels << @labels[label_name] - if issue.persisted? + if gitlab_issue.persisted? client.issue_comments(repo, issue.iid).each do |comment| # The note can be blank for issue service messages like "Changed title: ..." # We would like to import those comments as well but there is no any @@ -93,11 +97,11 @@ module Gitlab next unless comment.note.present? note = '' - note += @formatter.author_line(comment.author) unless existing_gitlab_user?(comment.author) + note += @formatter.author_line(comment.author) unless find_user_id(comment.author) note += comment.note begin - issue.notes.create!( + gitlab_issue.notes.create!( project: project, note: note, author_id: gitlab_user_id(project, comment.author), @@ -124,7 +128,7 @@ module Gitlab pull_requests.each do |pull_request| begin description = '' - description += @formatter.author_line(pull_request.author) unless existing_gitlab_user?(pull_request.author) + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) description += pull_request.description merge_request = project.merge_requests.create( -- cgit v1.2.1 From 170efaaba273792ddffc2806ef1501f33d87a5a2 Mon Sep 17 00:00:00 2001 From: Rydkin Maxim <maks.rydkin@gmail.com> Date: Fri, 16 Dec 2016 01:14:20 +0300 Subject: Enable Style/MultilineOperationIndentation in Rubocop, fixes #25741 --- lib/gitlab/email/reply_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 85402c2a278..f586c5ab062 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -69,7 +69,7 @@ module Gitlab # This one might be controversial but so many reply lines have years, times and end with a colon. # Let's try it and see how well it works. break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || - (l =~ /On \w+ \d+,? \d+,?.*wrote:/) + (l =~ /On \w+ \d+,? \d+,?.*wrote:/) # Headers on subsequent lines break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } -- cgit v1.2.1 From dbe2ac8ccc07f53669214eb954489a6cb233d4e9 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre <dbalexandre@gmail.com> Date: Fri, 16 Dec 2016 17:11:04 -0200 Subject: Fix rubucop offenses --- lib/gitlab/bitbucket_import/importer.rb | 54 ++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f3760640655..d5287c69e6e 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -60,8 +60,6 @@ module Gitlab create_labels - gitlab_issue = nil - client.issues(repo).each do |issue| begin description = '' @@ -87,31 +85,33 @@ module Gitlab gitlab_issue.labels << @labels[label_name] - if gitlab_issue.persisted? - client.issue_comments(repo, issue.iid).each do |comment| - # The note can be blank for issue service messages like "Changed title: ..." - # We would like to import those comments as well but there is no any - # specific parameter that would allow to process them, it's just an empty comment. - # To prevent our importer from just crashing or from creating useless empty comments - # we do this check. - next unless comment.note.present? - - note = '' - note += @formatter.author_line(comment.author) unless find_user_id(comment.author) - note += comment.note - - begin - gitlab_issue.notes.create!( - project: project, - note: note, - author_id: gitlab_user_id(project, comment.author), - created_at: comment.created_at, - updated_at: comment.updated_at - ) - rescue StandardError => e - errors << { type: :issue_comment, iid: issue.iid, errors: e.message } - end - end + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? + end + end + + def import_issue_comments(issue, gitlab_issue) + client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Changed title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + + note = '' + note += @formatter.author_line(comment.author) unless find_user_id(comment.author) + note += comment.note + + begin + gitlab_issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + rescue StandardError => e + errors << { type: :issue_comment, iid: issue.iid, errors: e.message } end end end -- cgit v1.2.1 From fe9a372c0b64b47117fc0a64dbdfb514f757ee6e Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre <dbalexandre@gmail.com> Date: Fri, 16 Dec 2016 19:11:48 -0200 Subject: Fix import issues method --- lib/gitlab/bitbucket_import/importer.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index d5287c69e6e..7d2f92d577a 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -79,13 +79,13 @@ module Gitlab created_at: issue.created_at, updated_at: issue.updated_at ) + + gitlab_issue.labels << @labels[label_name] + + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? rescue StandardError => e errors << { type: :issue, iid: issue.iid, errors: e.message } end - - gitlab_issue.labels << @labels[label_name] - - import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? end end -- cgit v1.2.1 From a3be4aeb7a71cc940394a5f13d09e79fcafdb1d5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre <dbalexandre@gmail.com> Date: Fri, 16 Dec 2016 19:51:40 -0200 Subject: Avoid use of Hash#dig to keep compatibility with Ruby 2.1 --- lib/bitbucket/representation/comment.rb | 2 +- lib/bitbucket/representation/issue.rb | 6 +++--- lib/bitbucket/representation/pull_request.rb | 10 +++++----- lib/bitbucket/representation/pull_request_comment.rb | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb index 3c75e9368fa..4937aa9728f 100644 --- a/lib/bitbucket/representation/comment.rb +++ b/lib/bitbucket/representation/comment.rb @@ -6,7 +6,7 @@ module Bitbucket end def note - raw.dig('content', 'raw') + raw.fetch('content', {}).fetch('raw', nil) end def created_at diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index 3af731753d1..054064395c3 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -12,11 +12,11 @@ module Bitbucket end def author - raw.dig('reporter', 'username') + raw.fetch('reporter', {}).fetch('username', nil) end def description - raw.dig('content', 'raw') + raw.fetch('content', {}).fetch('raw', nil) end def state @@ -28,7 +28,7 @@ module Bitbucket end def milestone - raw.dig('milestone', 'name') + raw['milestone']['name'] if raw['milestone'].present? end def created_at diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index e37c9a62c0e..eebf8093380 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -2,7 +2,7 @@ module Bitbucket module Representation class PullRequest < Representation::Base def author - raw.dig('author', 'username') + raw.fetch('author', {}).fetch('username', nil) end def description @@ -36,19 +36,19 @@ module Bitbucket end def source_branch_name - source_branch.dig('branch', 'name') + source_branch.fetch('branch', {}).fetch('name', nil) end def source_branch_sha - source_branch.dig('commit', 'hash') + source_branch.fetch('commit', {}).fetch('hash', nil) end def target_branch_name - target_branch.dig('branch', 'name') + target_branch.fetch('branch', {}).fetch('name', nil) end def target_branch_sha - target_branch.dig('commit', 'hash') + target_branch.fetch('commit', {}).fetch('hash', nil) end private diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index 4f3809fbcea..4f8efe03bae 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -18,7 +18,7 @@ module Bitbucket end def parent_id - raw.dig('parent', 'id') + raw.fetch('parent', {}).fetch('id', nil) end def inline? -- cgit v1.2.1 From f11caaf4692afdde0a2c458b3682aef3f9658b6a Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" <git@zjvandeweg.nl> Date: Mon, 12 Dec 2016 09:31:48 +0100 Subject: Setup mattermost session --- lib/mattermost/mattermost.rb | 102 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 lib/mattermost/mattermost.rb (limited to 'lib') diff --git a/lib/mattermost/mattermost.rb b/lib/mattermost/mattermost.rb new file mode 100644 index 00000000000..84d016bb197 --- /dev/null +++ b/lib/mattermost/mattermost.rb @@ -0,0 +1,102 @@ +module Mattermost + class NoSessionError < StandardError; end + # This class' prime objective is to obtain a session token on a Mattermost + # instance with SSO configured where this GitLab instance is the provider. + # + # The process depends on OAuth, but skips a step in the authentication cycle. + # For example, usually a user would click the 'login in GitLab' button on + # Mattermost, which would yield a 302 status code and redirects you to GitLab + # to approve the use of your account on Mattermost. Which would trigger a + # callback so Mattermost knows this request is approved and gets the required + # data to create the user account etc. + # + # This class however skips the button click, and also the approval phase to + # speed up the process and keep it without manual action and get a session + # going. + class Mattermost + include Doorkeeper::Helpers::Controller + include HTTParty + + attr_accessor :current_resource_owner + + def initialize(uri, current_user) + self.class.base_uri(uri) + + @current_resource_owner = current_user + end + + def with_session + raise NoSessionError unless create + yield + destroy + end + + # Next methods are needed for Doorkeeper + def pre_auth + @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new( + Doorkeeper.configuration, server.client_via_uid, params) + end + + def authorization + @authorization ||= strategy.request + end + + def strategy + @strategy ||= server.authorization_request(pre_auth.response_type) + end + + def request + @request ||= OpenStruct.new(parameters: params) + end + + def params + Rack::Utils.parse_query(@oauth_uri.query).symbolize_keys + end + + private + + def create + return unless oauth_uri + return unless token_uri + + self.class.headers("Cookie" => "MMAUTHTOKEN=#{request_token}") + + request_token + end + + def destroy + post('/api/v3/users/logout') + end + + def oauth_uri + response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) + return unless 300 <= response.code && response.code < 400 + + redirect_uri = response.headers['location'] + return unless redirect_uri + + @oauth_uri ||= URI.parse(redirect_uri) + end + + def token_uri + @token_uri ||= if @oauth_uri + authorization.authorize.redirect_uri if pre_auth.authorizable? + end + end + + def request_token + @request_token ||= if @token_uri + response = get(@token_uri, follow_redirects: false) + response.headers['token'] if 200 <= response.code && response.code < 400 + end + end + + def get(path, options = {}) + self.class.get(path, options) + end + + def post(path, options = {}) + self.class.post(path, options) + end + end +end -- cgit v1.2.1 From a31cdb29e49b62f0227963cbc54b6564a3ee9da8 Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" <git@zjvandeweg.nl> Date: Thu, 15 Dec 2016 14:32:50 +0100 Subject: Improve session tests --- lib/mattermost/mattermost.rb | 102 ----------------------------------------- lib/mattermost/session.rb | 105 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 102 deletions(-) delete mode 100644 lib/mattermost/mattermost.rb create mode 100644 lib/mattermost/session.rb (limited to 'lib') diff --git a/lib/mattermost/mattermost.rb b/lib/mattermost/mattermost.rb deleted file mode 100644 index 84d016bb197..00000000000 --- a/lib/mattermost/mattermost.rb +++ /dev/null @@ -1,102 +0,0 @@ -module Mattermost - class NoSessionError < StandardError; end - # This class' prime objective is to obtain a session token on a Mattermost - # instance with SSO configured where this GitLab instance is the provider. - # - # The process depends on OAuth, but skips a step in the authentication cycle. - # For example, usually a user would click the 'login in GitLab' button on - # Mattermost, which would yield a 302 status code and redirects you to GitLab - # to approve the use of your account on Mattermost. Which would trigger a - # callback so Mattermost knows this request is approved and gets the required - # data to create the user account etc. - # - # This class however skips the button click, and also the approval phase to - # speed up the process and keep it without manual action and get a session - # going. - class Mattermost - include Doorkeeper::Helpers::Controller - include HTTParty - - attr_accessor :current_resource_owner - - def initialize(uri, current_user) - self.class.base_uri(uri) - - @current_resource_owner = current_user - end - - def with_session - raise NoSessionError unless create - yield - destroy - end - - # Next methods are needed for Doorkeeper - def pre_auth - @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new( - Doorkeeper.configuration, server.client_via_uid, params) - end - - def authorization - @authorization ||= strategy.request - end - - def strategy - @strategy ||= server.authorization_request(pre_auth.response_type) - end - - def request - @request ||= OpenStruct.new(parameters: params) - end - - def params - Rack::Utils.parse_query(@oauth_uri.query).symbolize_keys - end - - private - - def create - return unless oauth_uri - return unless token_uri - - self.class.headers("Cookie" => "MMAUTHTOKEN=#{request_token}") - - request_token - end - - def destroy - post('/api/v3/users/logout') - end - - def oauth_uri - response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) - return unless 300 <= response.code && response.code < 400 - - redirect_uri = response.headers['location'] - return unless redirect_uri - - @oauth_uri ||= URI.parse(redirect_uri) - end - - def token_uri - @token_uri ||= if @oauth_uri - authorization.authorize.redirect_uri if pre_auth.authorizable? - end - end - - def request_token - @request_token ||= if @token_uri - response = get(@token_uri, follow_redirects: false) - response.headers['token'] if 200 <= response.code && response.code < 400 - end - end - - def get(path, options = {}) - self.class.get(path, options) - end - - def post(path, options = {}) - self.class.post(path, options) - end - end -end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb new file mode 100644 index 00000000000..d14121c91a0 --- /dev/null +++ b/lib/mattermost/session.rb @@ -0,0 +1,105 @@ +module Mattermost + class NoSessionError < StandardError; end + # This class' prime objective is to obtain a session token on a Mattermost + # instance with SSO configured where this GitLab instance is the provider. + # + # The process depends on OAuth, but skips a step in the authentication cycle. + # For example, usually a user would click the 'login in GitLab' button on + # Mattermost, which would yield a 302 status code and redirects you to GitLab + # to approve the use of your account on Mattermost. Which would trigger a + # callback so Mattermost knows this request is approved and gets the required + # data to create the user account etc. + # + # This class however skips the button click, and also the approval phase to + # speed up the process and keep it without manual action and get a session + # going. + class Session + include Doorkeeper::Helpers::Controller + include HTTParty + + attr_accessor :current_resource_owner + + def initialize(uri, current_user) + # Sets the base uri for HTTParty, so we can use paths + self.class.base_uri(uri) + + @current_resource_owner = current_user + end + + def with_session + raise NoSessionError unless create + result = yield + destroy + + result + end + + # Next methods are needed for Doorkeeper + def pre_auth + @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new( + Doorkeeper.configuration, server.client_via_uid, params) + end + + def authorization + @authorization ||= strategy.request + end + + def strategy + @strategy ||= server.authorization_request(pre_auth.response_type) + end + + def request + @request ||= OpenStruct.new(parameters: params) + end + + def params + Rack::Utils.parse_query(@oauth_uri.query).symbolize_keys + end + + private + + def create + return unless oauth_uri + return unless token_uri + + self.class.headers("Cookie" => "MMAUTHTOKEN=#{request_token}") + + request_token + end + + def destroy + post('/api/v3/users/logout') + end + + def oauth_uri + response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) + return unless 300 <= response.code && response.code < 400 + + redirect_uri = response.headers['location'] + return unless redirect_uri + + @oauth_uri ||= URI.parse(redirect_uri) + end + + def token_uri + @token_uri ||= if @oauth_uri + authorization.authorize.redirect_uri if pre_auth.authorizable? + end + end + + def request_token + @request_token ||= begin + response = get(@token_uri, follow_redirects: false) + response.headers['token'] if 200 <= response.code && response.code < 400 + end + end + + def get(path, options = {}) + self.class.get(path, options) + end + + def post(path, options = {}) + self.class.post(path, options) + end + end +end -- cgit v1.2.1 From 9bcc4d4de5510a14ae891105645b4d59891ba78d Mon Sep 17 00:00:00 2001 From: "Z.J. van de Weg" <git@zjvandeweg.nl> Date: Thu, 15 Dec 2016 21:06:17 +0100 Subject: Ensure the session is destroyed --- lib/mattermost/session.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index d14121c91a0..f4629585da7 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -28,10 +28,12 @@ module Mattermost def with_session raise NoSessionError unless create - result = yield - destroy - result + begin + yield + ensure + destroy + end end # Next methods are needed for Doorkeeper -- cgit v1.2.1 From 48ebfaa49146b8f6fcb24b063f22d553b2f20395 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski <ayufan@ayufan.eu> Date: Fri, 16 Dec 2016 11:31:26 +0100 Subject: Improve Mattermost Session specs --- lib/mattermost/session.rb | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index f4629585da7..7d0290be5a1 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -17,7 +17,7 @@ module Mattermost include Doorkeeper::Helpers::Controller include HTTParty - attr_accessor :current_resource_owner + attr_accessor :current_resource_owner, :token def initialize(uri, current_user) # Sets the base uri for HTTParty, so we can use paths @@ -64,9 +64,9 @@ module Mattermost return unless oauth_uri return unless token_uri - self.class.headers("Cookie" => "MMAUTHTOKEN=#{request_token}") - - request_token + self.token = request_token + self.class.headers("Cookie" => "MMAUTHTOKEN=#{self.token}") + self.token end def destroy @@ -84,16 +84,17 @@ module Mattermost end def token_uri - @token_uri ||= if @oauth_uri - authorization.authorize.redirect_uri if pre_auth.authorizable? - end + @token_uri ||= + if @oauth_uri + authorization.authorize.redirect_uri if pre_auth.authorizable? + end end def request_token - @request_token ||= begin - response = get(@token_uri, follow_redirects: false) - response.headers['token'] if 200 <= response.code && response.code < 400 - end + response = get(@token_uri, follow_redirects: false) + if 200 <= response.code && response.code < 400 + response.headers['token'] + end end def get(path, options = {}) -- cgit v1.2.1 From e663725961de66ac838d0a5a85978656938e74f4 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski <ayufan@ayufan.eu> Date: Fri, 16 Dec 2016 12:20:42 +0100 Subject: Store mattermost_url in settings --- lib/mattermost/session.rb | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 7d0290be5a1..a3715bed482 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -17,12 +17,11 @@ module Mattermost include Doorkeeper::Helpers::Controller include HTTParty - attr_accessor :current_resource_owner, :token + base_uri Settings.mattermost.host - def initialize(uri, current_user) - # Sets the base uri for HTTParty, so we can use paths - self.class.base_uri(uri) + attr_accessor :current_resource_owner, :token + def initialize(current_user) @current_resource_owner = current_user end @@ -30,7 +29,7 @@ module Mattermost raise NoSessionError unless create begin - yield + yield self ensure destroy end @@ -65,7 +64,9 @@ module Mattermost return unless token_uri self.token = request_token - self.class.headers("Cookie" => "MMAUTHTOKEN=#{self.token}") + @headers = { + "Authorization": "Bearer #{self.token}" + } self.token end @@ -98,11 +99,11 @@ module Mattermost end def get(path, options = {}) - self.class.get(path, options) + self.class.get(path, options.merge(headers: @headers)) end def post(path, options = {}) - self.class.post(path, options) + self.class.post(path, options.merge(headers: @headers)) end end end -- cgit v1.2.1 From c9610e0a052526adb3138dccf6114d710979a0b7 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski <ayufan@ayufan.eu> Date: Fri, 16 Dec 2016 13:43:01 +0100 Subject: Fix rubocop failures --- lib/mattermost/session.rb | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index a3715bed482..fb8d7d97f8a 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -54,7 +54,15 @@ module Mattermost end def params - Rack::Utils.parse_query(@oauth_uri.query).symbolize_keys + Rack::Utils.parse_query(oauth_uri.query).symbolize_keys + end + + def get(path, options = {}) + self.class.get(path, options.merge(headers: @headers)) + end + + def post(path, options = {}) + self.class.post(path, options.merge(headers: @headers)) end private @@ -63,11 +71,12 @@ module Mattermost return unless oauth_uri return unless token_uri - self.token = request_token + @token = request_token @headers = { - "Authorization": "Bearer #{self.token}" + Authorization: "Bearer #{@token}" } - self.token + + @token end def destroy @@ -75,35 +84,32 @@ module Mattermost end def oauth_uri + return @oauth_uri if defined?(@oauth_uri) + + @oauth_uri = nil + response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) return unless 300 <= response.code && response.code < 400 redirect_uri = response.headers['location'] return unless redirect_uri - @oauth_uri ||= URI.parse(redirect_uri) + @oauth_uri = URI.parse(redirect_uri) end def token_uri @token_uri ||= - if @oauth_uri + if oauth_uri authorization.authorize.redirect_uri if pre_auth.authorizable? end end def request_token - response = get(@token_uri, follow_redirects: false) + response = get(token_uri, follow_redirects: false) + if 200 <= response.code && response.code < 400 response.headers['token'] end end - - def get(path, options = {}) - self.class.get(path, options.merge(headers: @headers)) - end - - def post(path, options = {}) - self.class.post(path, options.merge(headers: @headers)) - end end end -- cgit v1.2.1