From db2433c36da6410c803163139e41228f9ae3f26b Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Tue, 2 Jan 2018 20:24:12 +0100 Subject: wip --- .../queries/additional_metrics_deployment_query.rb | 2 +- lib/gitlab/prometheus/queries/deployment_query.rb | 2 +- lib/gitlab/prometheus_client.rb | 36 ++++++++++------------ 3 files changed, 19 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 69d055c901c..294a6ae34ca 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,7 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics - def query(deployment_id) + def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( common_query_context( diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 170f483540e..6e6da593178 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,7 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery - def query(deployment_id) + def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index aa94614bf18..8ec4515fb12 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -3,10 +3,12 @@ module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient - attr_reader :api_url + attr_reader :api_url, :rest_client, :headers - def initialize(api_url:) + def initialize(api_url:, rest_client: nil, headers: nil) @api_url = api_url + @rest_client = rest_client || RestClient::Resource.new(api_url) + @headers = headers || {} end def ping @@ -40,24 +42,15 @@ module Gitlab private def json_api_get(type, args = {}) - get(join_api_url(type, args)) + path = ['api', 'v1', type].join('/') + get(path, args) rescue Errno::ECONNREFUSED raise PrometheusError, 'Connection refused' end - def join_api_url(type, args = {}) - url = URI.parse(api_url) - rescue URI::Error - raise PrometheusError, "Invalid API URL: #{api_url}" - else - url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/') - url.query = args.to_query - - url.to_s - end - - def get(url) - handle_response(HTTParty.get(url)) + def get(path, args) + response = rest_client[path].get(headers.merge(params: args)) + handle_response(response) rescue SocketError raise PrometheusError, "Can't connect to #{url}" rescue OpenSSL::SSL::SSLError @@ -67,15 +60,20 @@ module Gitlab end def handle_response(response) - if response.code == 200 && response['status'] == 'success' - response['data'] || {} + json_data = json_response(response) + if response.code == 200 && json_data['status'] == 'success' + json_data['data'] || {} elsif response.code == 400 - raise PrometheusError, response['error'] || 'Bad data received' + raise PrometheusError, json_data['error'] || 'Bad data received' else raise PrometheusError, "#{response.code} - #{response.body}" end end + def json_response(response) + JSON.parse(response.body) + end + def get_result(expected_type) data = yield data['result'] if data['resultType'] == expected_type -- cgit v1.2.1 From b38b5ceb8e039283e90dd323327e59c8f608c103 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Tue, 2 Jan 2018 21:42:24 +0100 Subject: Move client creation to Prometheus Application, manufacture proper rest client --- lib/gitlab/prometheus_client.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 8ec4515fb12..d1875dd1042 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -3,12 +3,10 @@ module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient - attr_reader :api_url, :rest_client, :headers + attr_reader :rest_client, :headers - def initialize(api_url:, rest_client: nil, headers: nil) - @api_url = api_url + def initialize(rest_client) @rest_client = rest_client || RestClient::Resource.new(api_url) - @headers = headers || {} end def ping @@ -49,7 +47,7 @@ module Gitlab end def get(path, args) - response = rest_client[path].get(headers.merge(params: args)) + response = rest_client[path].get(params: args) handle_response(response) rescue SocketError raise PrometheusError, "Can't connect to #{url}" @@ -60,7 +58,7 @@ module Gitlab end def handle_response(response) - json_data = json_response(response) + json_data = JSON.parse(response.body) if response.code == 200 && json_data['status'] == 'success' json_data['data'] || {} elsif response.code == 400 @@ -70,10 +68,6 @@ module Gitlab end end - def json_response(response) - JSON.parse(response.body) - end - def get_result(expected_type) data = yield data['result'] if data['resultType'] == expected_type -- cgit v1.2.1 From 80d4c0675f5715d724be20d47cafa372524e3ed1 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 4 Jan 2018 02:45:57 +0100 Subject: Add test checking if prometheus integration is enabled after prometheus is installed --- lib/gitlab/prometheus/queries/environment_query.rb | 26 ++++++++++------------ .../prometheus/queries/matched_metrics_query.rb | 2 +- 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 1d17d3cfd56..8f1453f31bf 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -2,22 +2,20 @@ module Gitlab module Prometheus module Queries class EnvironmentQuery < BaseQuery - def query(environment_id) - ::Environment.find_by(id: environment_id).try do |environment| - environment_slug = environment.slug - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f + def query + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f - memory_query = raw_memory_usage_query(environment_slug) - cpu_query = raw_cpu_usage_query(environment_slug) + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_current: client_query(memory_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_current: client_query(cpu_query, time: timeframe_end) - } - end + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index 4c3edccc71a..d21f64a252b 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -4,7 +4,7 @@ module Gitlab class MatchedMetricsQuery < BaseQuery MAX_QUERY_ITEMS = 40.freeze - def query + def query(_ = nil) groups_data.map do |group, data| { group: group.name, -- cgit v1.2.1 From 9c0b10da4147470050c1ae144fc75c2963eeb4ba Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 4 Jan 2018 03:13:54 +0100 Subject: Fix prometheus client tests --- lib/gitlab/prometheus_client.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index d1875dd1042..79f66b42c21 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -50,10 +50,12 @@ module Gitlab response = rest_client[path].get(params: args) handle_response(response) rescue SocketError - raise PrometheusError, "Can't connect to #{url}" + raise PrometheusError, "Can't connect to #{rest_client.url}" rescue OpenSSL::SSL::SSLError - raise PrometheusError, "#{url} contains invalid SSL data" - rescue HTTParty::Error + raise PrometheusError, "#{rest_client.url} contains invalid SSL data" + rescue RestClient::ExceptionWithResponse => ex + handle_exception_response(ex.response) + rescue RestClient::Exception raise PrometheusError, "Network connection error" end @@ -61,7 +63,14 @@ module Gitlab json_data = JSON.parse(response.body) if response.code == 200 && json_data['status'] == 'success' json_data['data'] || {} - elsif response.code == 400 + else + raise PrometheusError, "#{response.code} - #{response.body}" + end + end + + def handle_exception_response(response) + if response.code == 400 + json_data = JSON.parse(response.body) raise PrometheusError, json_data['error'] || 'Bad data received' else raise PrometheusError, "#{response.code} - #{response.body}" -- cgit v1.2.1 From e308bb0cd2226297093ffb488a4d97e3c02c4883 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 4 Jan 2018 13:56:07 +0100 Subject: Cleanup implementation and add cluster finding tests --- lib/gitlab/prometheus_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 79f66b42c21..8264501b1ae 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -6,7 +6,7 @@ module Gitlab attr_reader :rest_client, :headers def initialize(rest_client) - @rest_client = rest_client || RestClient::Resource.new(api_url) + @rest_client = rest_client end def ping -- cgit v1.2.1 From 720032733ad8fdb6124f2d0eee89807196552ad3 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Thu, 4 Jan 2018 15:04:14 +0100 Subject: Cleanup PrometheusService tests --- lib/gitlab/prometheus/queries/environment_query.rb | 26 ++++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 8f1453f31bf..1d17d3cfd56 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -2,20 +2,22 @@ module Gitlab module Prometheus module Queries class EnvironmentQuery < BaseQuery - def query - environment_slug = environment.slug - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f + def query(environment_id) + ::Environment.find_by(id: environment_id).try do |environment| + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f - memory_query = raw_memory_usage_query(environment_slug) - cpu_query = raw_cpu_usage_query(environment_slug) + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_current: client_query(memory_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_current: client_query(cpu_query, time: timeframe_end) - } + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end end end end -- cgit v1.2.1 From 1c0437d95e6329035af49d13b26c4a60948b02e3 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 2 Jan 2018 09:23:39 +0100 Subject: Client implementation for WikiPageVerion Part of gitlab-org/gitaly#639 --- lib/gitlab/git/wiki.rb | 38 ++++++++++++++++------ lib/gitlab/gitaly_client/wiki_service.rb | 56 +++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index d4a53d32c28..305bc9b66b0 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -93,11 +93,23 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - current_page = gollum_page_by_path(page_path) + puts '-' * 80 + puts options + puts '-' * 80 + puts - commits_from_page(current_page, options).map do |gitlab_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) - Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) + byebug + @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled| + if is_enabled + gitaly_wiki_client.page_versions(page_path, pagination_params(options)) + else + current_page = gollum_page_by_path(page_path) + + commits_from_page(current_page, options).map do |gitlab_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) + Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) + end + end end end @@ -124,15 +136,21 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def commits_from_page(gollum_page, options = {}) - unless options[:limit] - options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page - options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i - end + pagination_options = pagination_params(options) @repository.log(ref: gollum_page.last_version.id, path: gollum_page.path, - limit: options[:limit], - offset: options[:offset]) + limit: pagination_options[:limit], + offset: pagination_options[:offset]) + end + + def pagination_params(options) + return options if options[:limit] + + options = options.dup + options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page + options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i + options end def gollum_wiki diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 5c5b170a3e0..fa7e416504d 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -101,6 +101,48 @@ module Gitlab pages end + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def page_versions(page_path, options) + request = Gitaly::WikiGetPageVersionsRequest.new( + repository: @gitaly_repo, + page_path: encode_binary(page_path) + ) + + min_index = options[:offset].to_i + max_index = min_index + options[:limit].to_i + byebug + + stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request) + + version_index = 0 + versions = [] + + # Allow limit and offset to be send to Gitaly: TODO + stream.each do |message| + puts '§' * 80 + puts version_index + + message.versions.each do |version| + case version_index + when min_index...max_index + versions << new_wiki_page_version(version) + when max_index..Float::INFINITY + return versions + end + + version_index += 1 + puts version_index + end + end + + # when we're requesting page 99 but the stream doesn't go that far, whatever + # is fetched thus far + versions + end + def find_file(name, revision) request = Gitaly::WikiFindFileRequest.new( repository: @gitaly_repo, @@ -129,7 +171,7 @@ module Gitlab private - # If a block is given and the yielded value is true, iteration will be + # If a block is given and the yielded value is truthy, iteration will be # stopped early at that point; else the iterator is consumed entirely. # The iterator is traversed with `next` to allow resuming the iteration. def wiki_page_from_iterator(iterator) @@ -146,10 +188,7 @@ module Gitlab else wiki_page = GitalyClient::WikiPage.new(page.to_h) - version = Gitlab::Git::WikiPageVersion.new( - Gitlab::Git::Commit.decorate(@repository, page.version.commit), - page.version.format - ) + version = new_wiki_page_version(page.version) end end @@ -158,6 +197,13 @@ module Gitlab [wiki_page, version] end + def new_wiki_page_version(version) + Gitlab::Git::WikiPageVersion.new( + Gitlab::Git::Commit.decorate(@repository, version.commit), + version.format + ) + end + def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( name: encode_binary(commit_details.name), -- cgit v1.2.1 From a29f0c28fd07ba14f0d0e5fb9c878a2eb117e388 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 16 Jan 2018 11:12:08 +0100 Subject: Allow pagination for WikiVersions on Gitaly request --- lib/gitlab/git/wiki.rb | 32 +++++++++++++------------------- lib/gitlab/gitaly_client/wiki_service.rb | 26 ++++---------------------- 2 files changed, 17 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 305bc9b66b0..4ed78daa443 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -93,15 +93,15 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - puts '-' * 80 - puts options - puts '-' * 80 - puts - - byebug @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled| if is_enabled - gitaly_wiki_client.page_versions(page_path, pagination_params(options)) + versions = gitaly_wiki_client.page_versions(page_path, options) + + # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 + # per page, but also fetches 20 if `limit` or `per_page` < 20. + # Slicing returns an array with the expected number of items. + slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page + versions[0..slice_bound] else current_page = gollum_page_by_path(page_path) @@ -136,21 +136,15 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def commits_from_page(gollum_page, options = {}) - pagination_options = pagination_params(options) + unless options[:limit] + options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page + options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i + end @repository.log(ref: gollum_page.last_version.id, path: gollum_page.path, - limit: pagination_options[:limit], - offset: pagination_options[:offset]) - end - - def pagination_params(options) - return options if options[:limit] - - options = options.dup - options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page - options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i - options + limit: options[:limit], + offset: options[:offset]) end def gollum_wiki diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index fa7e416504d..a98c3c0b160 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -108,38 +108,20 @@ module Gitlab def page_versions(page_path, options) request = Gitaly::WikiGetPageVersionsRequest.new( repository: @gitaly_repo, - page_path: encode_binary(page_path) + page_path: encode_binary(page_path), + page: options[:page] || 1, + per_page: options[:per_page] || Gollum::Page.per_page ) - min_index = options[:offset].to_i - max_index = min_index + options[:limit].to_i - byebug - stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request) - version_index = 0 versions = [] - - # Allow limit and offset to be send to Gitaly: TODO stream.each do |message| - puts '§' * 80 - puts version_index - message.versions.each do |version| - case version_index - when min_index...max_index - versions << new_wiki_page_version(version) - when max_index..Float::INFINITY - return versions - end - - version_index += 1 - puts version_index + versions << new_wiki_page_version(version) end end - # when we're requesting page 99 but the stream doesn't go that far, whatever - # is fetched thus far versions end -- cgit v1.2.1 From b1c40590e7738a782d1295d743a6a3957723c4b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 29 Jan 2018 00:23:36 +0100 Subject: Extend Runner API helpers with cache info storage --- lib/api/helpers/runner.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 2cae53dba53..ad8c1c4407c 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -27,6 +27,8 @@ module API end def update_runner_info + update_runner_info_cache + return unless update_runner? current_runner.contacted_at = Time.now @@ -35,13 +37,17 @@ module API end def update_runner? - # Use a random threshold to prevent beating DB updates. - # It generates a distribution between [40m, 80m]. - # - contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + # Use a 1h threshold to prevent beating DB updates. current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= contacted_at_max_age + (Time.now - current_runner.contacted_at) >= UPDATE_RUNNER_EVERY + end + + def update_runner_info_cache + Gitlab::Redis::SharedState.with do |redis| + redis_key = "#{current_runner.runner_info_key}:contacted_at" + redis.set(redis_key, Time.now) + end end def validate_job!(job) -- cgit v1.2.1 From d90d141c24228b8df6333b03d26a1723480837ba Mon Sep 17 00:00:00 2001 From: Tony Rom Date: Wed, 20 Dec 2017 14:39:40 +0300 Subject: Add Colors to GitLab Flavored Markdown --- lib/banzai/color_parser.rb | 50 +++++++++++++++++++++++ lib/banzai/filter/color_filter.rb | 31 ++++++++++++++ lib/banzai/pipeline/broadcast_message_pipeline.rb | 1 + lib/banzai/pipeline/gfm_pipeline.rb | 1 + 4 files changed, 83 insertions(+) create mode 100644 lib/banzai/color_parser.rb create mode 100644 lib/banzai/filter/color_filter.rb (limited to 'lib') diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb new file mode 100644 index 00000000000..d96c0a1ed1f --- /dev/null +++ b/lib/banzai/color_parser.rb @@ -0,0 +1,50 @@ +module Banzai + module ColorParser + ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0 + PERCENTS = /(?:\d{1,2}|100)%/ # 00%..100% + ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/ + BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/ # 00..255 + DEGS = /-?\d+(?:deg)?/i # [-]digits[deg] + RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad + HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/ + RGB_FORMAT = / + (?:rgba? + \( + (?: + (?:(?:#{BITS},\s*){2}#{BITS}) + | + (?:(?:#{PERCENTS},\s*){2}#{PERCENTS}) + ) + #{ALPHA_CHANNEL} + \) + ) + /xi + HSL_FORMAT = / + (?:hsla? + \( + (?:#{DEGS}|#{RADS}),\s*#{PERCENTS},\s*#{PERCENTS} + #{ALPHA_CHANNEL} + \) + ) + /xi + + FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze + + class << self + # Public: Analyzes whether the String is a color code. + # + # text - The String to be parsed. + # + # Returns the recognized color String or nil if none was found. + def parse(text) + text if color_format =~ text + end + + private + + def color_format + @color_format ||= /\A(#{Regexp.union(FORMATS)})\z/ix + end + end + end +end diff --git a/lib/banzai/filter/color_filter.rb b/lib/banzai/filter/color_filter.rb new file mode 100644 index 00000000000..6ab29ac281f --- /dev/null +++ b/lib/banzai/filter/color_filter.rb @@ -0,0 +1,31 @@ +module Banzai + module Filter + # HTML filter that renders `color` followed by a color "chip". + # + class ColorFilter < HTML::Pipeline::Filter + COLOR_CHIP_CLASS = 'gfm-color_chip'.freeze + + def call + doc.css('code').each do |node| + color = ColorParser.parse(node.content) + node << color_chip(color) if color + end + + doc + end + + private + + def color_chip(color) + checkerboard = doc.document.create_element('span', class: COLOR_CHIP_CLASS) + chip = doc.document.create_element('span', style: inline_styles(color: color)) + + checkerboard << chip + end + + def inline_styles(color:) + "background-color: #{color};" + end + end + end +end diff --git a/lib/banzai/pipeline/broadcast_message_pipeline.rb b/lib/banzai/pipeline/broadcast_message_pipeline.rb index adc09c8afbd..5dd572de3a1 100644 --- a/lib/banzai/pipeline/broadcast_message_pipeline.rb +++ b/lib/banzai/pipeline/broadcast_message_pipeline.rb @@ -7,6 +7,7 @@ module Banzai Filter::SanitizationFilter, Filter::EmojiFilter, + Filter::ColorFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter ] diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index c746f6f64e9..4001b8a85e3 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -14,6 +14,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::MathFilter, + Filter::ColorFilter, Filter::MermaidFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, -- cgit v1.2.1 From 397442a06140a8cf38bebe3f4519311b1448f23d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 29 Jan 2018 17:21:30 +0100 Subject: Update runner info on all authenticated requests --- lib/api/helpers/runner.rb | 2 ++ lib/api/runner.rb | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index ad8c1c4407c..5ac9181f878 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -20,6 +20,8 @@ module API def authenticate_runner! forbidden! unless current_runner + + update_runner_info end def current_runner diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 80feb629d54..50cb1df92ad 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -78,7 +78,6 @@ module API post '/request' do authenticate_runner! no_content! unless current_runner.active? - update_runner_info if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] -- cgit v1.2.1 From bdd3e39b0bd4e8fcec5d6e2d97297840211dd921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Mon, 29 Jan 2018 20:56:28 +0100 Subject: Move info update implementation to Ci::Runner model --- lib/api/helpers/runner.rb | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 5ac9181f878..8f45cae0e60 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -5,7 +5,6 @@ module API JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 10 * 60 def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], @@ -21,37 +20,13 @@ module API def authenticate_runner! forbidden! unless current_runner - update_runner_info + current_runner.update_runner_info(get_runner_version_from_params) end def current_runner @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) end - def update_runner_info - update_runner_info_cache - - return unless update_runner? - - current_runner.contacted_at = Time.now - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - - def update_runner? - # Use a 1h threshold to prevent beating DB updates. - - current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= UPDATE_RUNNER_EVERY - end - - def update_runner_info_cache - Gitlab::Redis::SharedState.with do |redis| - redis_key = "#{current_runner.runner_info_key}:contacted_at" - redis.set(redis_key, Time.now) - end - end - def validate_job!(job) not_found! unless job -- cgit v1.2.1 From a248e7adf3c11bf9e0231570163930c953a1a8ee Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 31 Jan 2018 13:41:14 +0100 Subject: Fix flay not detecting identical code Previously the script checked only for 'Similar code', but flay will tell users when IDENTICAL code is detected too. This should create a commit will create a failing pipeline, and to check that it does this commit is pushed. Fixes #42628 --- lib/tasks/flay.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index b1e012e70c5..4b4881cecb8 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -2,7 +2,7 @@ desc 'Code duplication analyze via flay' task :flay do output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}` - if output.include? "Similar code found" + if output.include?("Similar code found") || output.include?("IDENTICAL code found") puts output exit 1 end -- cgit v1.2.1 From 57f8b24fe2d54df4dfcc3ddcb5796e41093f0fd7 Mon Sep 17 00:00:00 2001 From: Tony Rom Date: Thu, 1 Feb 2018 13:35:03 +0300 Subject: Fix minors --- lib/banzai/color_parser.rb | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb index d96c0a1ed1f..0cd9085190c 100644 --- a/lib/banzai/color_parser.rb +++ b/lib/banzai/color_parser.rb @@ -30,21 +30,15 @@ module Banzai FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze - class << self - # Public: Analyzes whether the String is a color code. - # - # text - The String to be parsed. - # - # Returns the recognized color String or nil if none was found. - def parse(text) - text if color_format =~ text - end + COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix - private - - def color_format - @color_format ||= /\A(#{Regexp.union(FORMATS)})\z/ix - end + # Public: Analyzes whether the String is a color code. + # + # text - The String to be parsed. + # + # Returns the recognized color String or nil if none was found. + def self.parse(text) + text if COLOR_FORMAT =~ text end end end -- cgit v1.2.1 From 0a47d1924d6b283a174672f33cf7a0de6b281fef Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 30 Jan 2018 09:59:45 +0100 Subject: Client changes for Tag,BranchNamesContainingCommit As part of gitlab-org/gitaly#884, this commit contains the client implementation for both TagNamesContaintingCommit and BranchNamesContainingCommit. The interface in the Repository model stays the same, but the implementation on the serverside, e.g. Gitaly, uses `for-each-ref`, as opposed to `branch` or `tag` which both aren't plumbing command. The result stays the same. On the serverside, we have the opportunity to limit the number of names to return. However, this is not supported on the front end yet. My proposal to use this ability: gitlab-org/gitlab-ce#42581. For now, this ability is not used as that would change more behaviours on a feature flag which might lead to unexpected changes on page refresh for example. --- lib/gitlab/git/branch.rb | 18 ++++++++------ lib/gitlab/git/repository.rb | 42 +++++++++++++++++++++++---------- lib/gitlab/git/tag.rb | 8 +++++-- lib/gitlab/gitaly_client/ref_service.rb | 26 ++++++++++++++++++++ 4 files changed, 73 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 3487e099381..2c051bd33b9 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,13 +1,17 @@ -# Gitaly note: JV: no RPC's here. - module Gitlab module Git class Branch < Ref - def self.find(repo, branch_name) - if branch_name.is_a?(Gitlab::Git::Branch) - branch_name - else - repo.find_branch(branch_name) + class << self + def find(repo, branch_name) + if branch_name.is_a?(Gitlab::Git::Branch) + branch_name + else + repo.find_branch(branch_name) + end + end + + def names_contains_sha(repo, sha, limit: 0) + GitalyClient::RefService.new(repo).branch_names_contains_sha(sha) end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 68b54d28876..76b94bf9030 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1355,20 +1355,23 @@ module Gitlab raise CommandError.new(e) end - def refs_contains_sha(ref_type, sha) - args = %W(#{ref_type} --contains #{sha}) - names = run_git(args).first - - if names.respond_to?(:split) - names = names.split("\n").map(&:strip) - - names.each do |name| - name.slice! '* ' + def branch_names_contains_sha(sha) + gitaly_migrate(:branch_names_contains_sha) do |is_enabled| + if is_enabled + Gitlab::Git::Branch.names_contains_sha(self, sha) + else + refs_contains_sha(:branch, sha) end + end + end - names - else - [] + def tag_names_contains_sha(sha) + gitaly_migrate(:tag_names_contains_sha) do |is_enabled| + if is_enabled + Gitlab::Git::Tag.names_contains_sha(self, sha) + else + refs_contains_sha(:tag, sha) + end end end @@ -1446,6 +1449,21 @@ module Gitlab end end + def refs_contains_sha(ref_type, sha) + args = %W(#{ref_type} --contains #{sha}) + names = run_git(args).first + + return [] unless names.respond_to?(:split) + + names = names.split("\n").map(&:strip) + + names.each do |name| + name.slice! '* ' + end + + names + end + def shell_write_ref(ref_path, ref, old_ref) raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index bc4e160dce9..c163d383eb0 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,8 +1,12 @@ -# Gitaly note: JV: no RPC's here. -# module Gitlab module Git class Tag < Ref + class << self + def names_contains_sha(repo, sha) + GitalyClient::RefService.new(repo).branch_names_contains_sha(sha) + end + end + attr_reader :object_sha def initialize(repository, name, target, target_commit, message = nil) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8b9a224b700..07122da4c2c 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -145,6 +145,32 @@ module Gitlab raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end + # Limit: 0 implies no limit, thus all tag names will be returned + def tag_names_containing(sha, limit: 0) + request = Gitaly::ListTagNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) + + stream.each_with_object([]) { |response, array| array.concat(response.tag_names) } + end + + # Limit: 0 implies no limit, thus all tag names will be returned + def branch_names_contains_sha(sha, limit: 0) + request = Gitaly::ListBranchNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) + + stream.each_with_object([]) { |response, array| array.concat(response.branch_names) } + end + private def consume_refs_response(response) -- cgit v1.2.1 From 73bd48de977bea1c8cd8bb3d995b984b874d4eba Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 30 Jan 2018 10:56:24 +0100 Subject: Fix encoding issues when name is not UTF-8 --- lib/gitlab/gitaly_client/ref_service.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 07122da4c2c..22fee1ff07f 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -155,7 +155,10 @@ module Gitlab stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) - stream.each_with_object([]) { |response, array| array.concat(response.tag_names) } + stream.each_with_object([]) do |response, array| + encoded_names = response.tag_names.map { |t| Gitlab::Git.ref_name(t) } + array.concat(encoded_names) + end end # Limit: 0 implies no limit, thus all tag names will be returned @@ -168,7 +171,10 @@ module Gitlab stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) - stream.each_with_object([]) { |response, array| array.concat(response.branch_names) } + stream.each_with_object([]) do |response, array| + encoded_names = response.branch_names.map { |b| Gitlab::Git.ref_name(b) } + array.concat(encoded_names) + end end private -- cgit v1.2.1 From fd46d6ceb81eb9039b4e60c1d158848dd22ba411 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 30 Jan 2018 11:22:08 +0100 Subject: Remove intermediate methods on Branch and Tag classes --- lib/gitlab/git/branch.rb | 16 +++++----------- lib/gitlab/git/repository.rb | 4 ++-- lib/gitlab/git/tag.rb | 6 ------ lib/gitlab/gitaly_client/ref_service.rb | 19 ++++++++++--------- 4 files changed, 17 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 2c051bd33b9..ae7e88f0503 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,17 +1,11 @@ module Gitlab module Git class Branch < Ref - class << self - def find(repo, branch_name) - if branch_name.is_a?(Gitlab::Git::Branch) - branch_name - else - repo.find_branch(branch_name) - end - end - - def names_contains_sha(repo, sha, limit: 0) - GitalyClient::RefService.new(repo).branch_names_contains_sha(sha) + def self.find(repo, branch_name) + if branch_name.is_a?(Gitlab::Git::Branch) + branch_name + else + repo.find_branch(branch_name) end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 76b94bf9030..95644dacc4e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1358,7 +1358,7 @@ module Gitlab def branch_names_contains_sha(sha) gitaly_migrate(:branch_names_contains_sha) do |is_enabled| if is_enabled - Gitlab::Git::Branch.names_contains_sha(self, sha) + gitaly_ref_client.branch_names_contains_sha(sha) else refs_contains_sha(:branch, sha) end @@ -1368,7 +1368,7 @@ module Gitlab def tag_names_contains_sha(sha) gitaly_migrate(:tag_names_contains_sha) do |is_enabled| if is_enabled - Gitlab::Git::Tag.names_contains_sha(self, sha) + gitaly_ref_client.tag_names_contains_sha(sha) else refs_contains_sha(:tag, sha) end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index c163d383eb0..8a8f7b051ed 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,12 +1,6 @@ module Gitlab module Git class Tag < Ref - class << self - def names_contains_sha(repo, sha) - GitalyClient::RefService.new(repo).branch_names_contains_sha(sha) - end - end - attr_reader :object_sha def initialize(repository, name, target, target_commit, message = nil) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 22fee1ff07f..ba6b577fd17 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -146,7 +146,7 @@ module Gitlab end # Limit: 0 implies no limit, thus all tag names will be returned - def tag_names_containing(sha, limit: 0) + def tag_names_contains_sha(sha, limit: 0) request = Gitaly::ListTagNamesContainingCommitRequest.new( repository: @gitaly_repo, commit_id: sha, @@ -155,10 +155,7 @@ module Gitlab stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) - stream.each_with_object([]) do |response, array| - encoded_names = response.tag_names.map { |t| Gitlab::Git.ref_name(t) } - array.concat(encoded_names) - end + consume_ref_contains_sha_response(stream, :tag_names) end # Limit: 0 implies no limit, thus all tag names will be returned @@ -171,10 +168,7 @@ module Gitlab stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) - stream.each_with_object([]) do |response, array| - encoded_names = response.branch_names.map { |b| Gitlab::Git.ref_name(b) } - array.concat(encoded_names) - end + consume_ref_contains_sha_response(stream, :branch_names) end private @@ -247,6 +241,13 @@ module Gitlab Gitlab::Git::Commit.decorate(@repository, hash) end + def consume_ref_contains_sha_response(stream, collection_name) + stream.each_with_object([]) do |response, array| + encoded_names = response.send(collection_name).map { |b| Gitlab::Git.ref_name(b) } # rubocop:disable GitlabSecurity/PublicSend + array.concat(encoded_names) + end + end + def invalid_ref!(message) raise Gitlab::Git::Repository::InvalidRef.new(message) end -- cgit v1.2.1 From cca61980d5ad9c4db65b9498fe49d936657bc0e2 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 15 Jan 2018 16:21:04 +0100 Subject: Track and act upon the number of executed queries This ensures that we have more visibility in the number of SQL queries that are executed in web requests. The current threshold is hardcoded to 100 as we will rarely (maybe once or twice) change it. In production and development we use Sentry if enabled, in the test environment we raise an error. This feature is also only enabled in production/staging when running on GitLab.com as it's not very useful to other users. --- lib/api/branches.rb | 2 + lib/api/issues.rb | 6 ++ lib/api/merge_requests.rb | 6 ++ lib/api/pipelines.rb | 2 + lib/api/projects.rb | 2 + lib/api/triggers.rb | 2 + lib/api/users.rb | 2 + lib/api/v3/branches.rb | 2 + lib/api/v3/issues.rb | 6 ++ lib/api/v3/merge_requests.rb | 4 ++ lib/api/v3/pipelines.rb | 2 + lib/api/v3/triggers.rb | 2 + lib/gitlab/query_limiting.rb | 36 ++++++++++ .../query_limiting/active_support_subscriber.rb | 11 +++ lib/gitlab/query_limiting/middleware.rb | 55 ++++++++++++++ lib/gitlab/query_limiting/transaction.rb | 83 ++++++++++++++++++++++ 16 files changed, 223 insertions(+) create mode 100644 lib/gitlab/query_limiting.rb create mode 100644 lib/gitlab/query_limiting/active_support_subscriber.rb create mode 100644 lib/gitlab/query_limiting/middleware.rb create mode 100644 lib/gitlab/query_limiting/transaction.rb (limited to 'lib') diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 0791a110c39..1794207e29b 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -29,6 +29,8 @@ module API use :pagination end get ':id/repository/branches' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329') + repository = user_project.repository branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name)) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index c99fe3ab5b3..b6c278c89d0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -161,6 +161,8 @@ module API use :issue_params end post ':id/issues' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42320') + authorize! :create_issue, user_project # Setting created_at time only allowed for admins and project owners @@ -201,6 +203,8 @@ module API :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322') + issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue @@ -234,6 +238,8 @@ module API requires :to_project_id, type: Integer, desc: 'The ID of the new project' end post ':id/issues/:issue_iid/move' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323') + issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 420aaf1c964..719afa09295 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -152,6 +152,8 @@ module API use :optional_params end post ":id/merge_requests" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42316') + authorize! :create_merge_request, user_project mr_params = declared_params(include_missing: false) @@ -256,6 +258,8 @@ module API at_least_one_of(*at_least_one_of_ce) end put ':id/merge_requests/:merge_request_iid' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42318') + merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) mr_params = declared_params(include_missing: false) @@ -283,6 +287,8 @@ module API optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' end put ':id/merge_requests/:merge_request_iid/merge' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42317') + merge_request = find_project_merge_request(params[:merge_request_iid]) merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 675c963bae2..d2b8b832e4e 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -42,6 +42,8 @@ module API requires :ref, type: String, desc: 'Reference' end post ':id/pipeline' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42124') + authorize! :create_pipeline, user_project new_pipeline = Ci::CreatePipelineService.new(user_project, diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 8b5e4f8edcc..5b481121a10 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -210,6 +210,8 @@ module API optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' end post ':id/fork' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42284') + fork_params = declared_params(include_missing: false) namespace_id = fork_params[:namespace] diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index dd6801664b1..b3709455bc3 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -15,6 +15,8 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42283') + # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } diff --git a/lib/api/users.rb b/lib/api/users.rb index e5de31ad51b..c7c2aa280d5 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -383,6 +383,8 @@ module API optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end delete ":id" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42279') + authenticated_as_admin! user = User.find_by(id: params[:id]) diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb index b201bf77667..25176c5b38e 100644 --- a/lib/api/v3/branches.rb +++ b/lib/api/v3/branches.rb @@ -14,6 +14,8 @@ module API success ::API::Entities::Branch end get ":id/repository/branches" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42276') + repository = user_project.repository branches = repository.branches.sort_by(&:name) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb index cb371fdbab8..b59947d81d9 100644 --- a/lib/api/v3/issues.rb +++ b/lib/api/v3/issues.rb @@ -134,6 +134,8 @@ module API use :issue_params end post ':id/issues' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42131') + # Setting created_at time only allowed for admins and project owners unless current_user.admin? || user_project.owner == current_user params.delete(:created_at) @@ -169,6 +171,8 @@ module API :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_id' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42132') + issue = user_project.issues.find(params.delete(:issue_id)) authorize! :update_issue, issue @@ -201,6 +205,8 @@ module API requires :to_project_id, type: Integer, desc: 'The ID of the new project' end post ':id/issues/:issue_id/move' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42133') + issue = user_project.issues.find_by(id: params[:issue_id]) not_found!('Issue') unless issue diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 0a24fea52a3..ce216497996 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -91,6 +91,8 @@ module API use :optional_params end post ":id/merge_requests" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42126') + authorize! :create_merge_request, user_project mr_params = declared_params(include_missing: false) @@ -167,6 +169,8 @@ module API :remove_source_branch end put path do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42127') + merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) mr_params = declared_params(include_missing: false) diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb index c48cbd2b765..6d31c12f572 100644 --- a/lib/api/v3/pipelines.rb +++ b/lib/api/v3/pipelines.rb @@ -19,6 +19,8 @@ module API desc: 'Either running, branches, or tags' end get ':id/pipelines' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42123') + authorize! :read_pipeline, user_project pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index 534911fde5c..34f07dfb486 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -16,6 +16,8 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42121') + # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb new file mode 100644 index 00000000000..f64f1757144 --- /dev/null +++ b/lib/gitlab/query_limiting.rb @@ -0,0 +1,36 @@ +module Gitlab + module QueryLimiting + # Returns true if we should enable tracking of query counts. + # + # This is only enabled in production/staging if we're running on GitLab.com. + # This ensures we don't produce any errors that users can't do anything + # about themselves. + def self.enable? + Gitlab.com? || Rails.env.development? || Rails.env.test? + end + + # Allows the current request to execute any number of SQL queries. + # + # This method should _only_ be used when there's a corresponding issue to + # reduce the number of queries. + # + # The issue URL is only meant to push developers into creating an issue + # instead of blindly whitelisting offending blocks of code. + def self.whitelist(issue_url) + return unless enable_whitelist? + + unless issue_url.start_with?('https://') + raise( + ArgumentError, + 'You must provide a valid issue URL in order to whitelist a block of code' + ) + end + + Transaction&.current&.whitelisted = true + end + + def self.enable_whitelist? + Rails.env.development? || Rails.env.test? + end + end +end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb new file mode 100644 index 00000000000..66049c94ec6 --- /dev/null +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -0,0 +1,11 @@ +module Gitlab + module QueryLimiting + class ActiveSupportSubscriber < ActiveSupport::Subscriber + attach_to :active_record + + def sql(*) + Transaction.current&.increment + end + end + end +end diff --git a/lib/gitlab/query_limiting/middleware.rb b/lib/gitlab/query_limiting/middleware.rb new file mode 100644 index 00000000000..949ae79a047 --- /dev/null +++ b/lib/gitlab/query_limiting/middleware.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module QueryLimiting + # Middleware for reporting (or raising) when a request performs more than a + # certain amount of database queries. + class Middleware + CONTROLLER_KEY = 'action_controller.instance'.freeze + ENDPOINT_KEY = 'api.endpoint'.freeze + + def initialize(app) + @app = app + end + + def call(env) + transaction, retval = Transaction.run do + @app.call(env) + end + + transaction.action = action_name(env) + transaction.act_upon_results + + retval + end + + def action_name(env) + if env[CONTROLLER_KEY] + action_for_rails(env) + elsif env[ENDPOINT_KEY] + action_for_grape(env) + end + end + + private + + def action_for_rails(env) + controller = env[CONTROLLER_KEY] + action = "#{controller.class.name}##{controller.action_name}" + + if controller.content_type == 'text/html' + action + else + "#{action} (#{controller.content_type})" + end + end + + def action_for_grape(env) + endpoint = env[ENDPOINT_KEY] + route = endpoint.route rescue nil + + "#{route.request_method} #{route.path}" if route + end + end + end +end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb new file mode 100644 index 00000000000..3cbafadb0d0 --- /dev/null +++ b/lib/gitlab/query_limiting/transaction.rb @@ -0,0 +1,83 @@ +module Gitlab + module QueryLimiting + class Transaction + THREAD_KEY = :__gitlab_query_counts_transaction + + attr_accessor :count, :whitelisted + + # The name of the action (e.g. `UsersController#show`) that is being + # executed. + attr_accessor :action + + # The maximum number of SQL queries that can be executed in a request. For + # the sake of keeping things simple we hardcode this value here, it's not + # supposed to be changed very often anyway. + THRESHOLD = 100 + + # Error that is raised whenever exceeding the maximum number of queries. + ThresholdExceededError = Class.new(StandardError) + + def self.current + Thread.current[THREAD_KEY] + end + + # Starts a new transaction and returns it and the blocks' return value. + # + # Example: + # + # transaction, retval = Transaction.run do + # 10 + # end + # + # retval # => 10 + def self.run + transaction = new + Thread.current[THREAD_KEY] = transaction + + [transaction, yield] + ensure + Thread.current[THREAD_KEY] = nil + end + + def initialize + @action = nil + @count = 0 + @whitelisted = false + end + + # Sends a notification based on the number of executed SQL queries. + def act_upon_results + return unless threshold_exceeded? + + error = ThresholdExceededError.new(error_message) + + if raise_error? + raise(error) + else + # Raven automatically logs to the Rails log if disabled, thus we don't + # need to manually log anything in case Sentry support is not enabled. + Raven.capture_exception(error) + end + end + + def increment + @count += 1 unless whitelisted + end + + def raise_error? + Rails.env.test? + end + + def threshold_exceeded? + count > THRESHOLD + end + + def error_message + header = 'Too many SQL queries were executed' + header += " in #{action}" if action + + "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + end + end + end +end -- cgit v1.2.1 From 972f564dcd897ee66799c19f96ed22df588f8a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Thu, 1 Feb 2018 17:00:16 -0500 Subject: Sanitize extra blank spaces used when uploading a SSH key --- lib/gitlab/ssh_public_key.rb | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 89ca1298120..545e7c74f7e 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -21,6 +21,22 @@ module Gitlab technology(name)&.supported_sizes end + def self.sanitize(key_content) + ssh_type, *parts = key_content.strip.split + + return key_content if parts.empty? + + parts.each_with_object("#{ssh_type} ").with_index do |(part, content), index| + content << part + + if Gitlab::SSHPublicKey.new(content).valid? + break [content, parts[index + 1]].compact.join(' ') # Add the comment part if present + elsif parts.size == index + 1 # return original content if we've reached the last element + break key_content + end + end + end + attr_reader :key_text, :key # Unqualified MD5 fingerprint for compatibility @@ -37,23 +53,23 @@ module Gitlab end def valid? - key.present? + key.present? && bits && technology.supported_sizes.include?(bits) end def type - technology.name if valid? + technology.name if key.present? end def bits - return unless valid? + return if key.blank? case type when :rsa - key.n.num_bits + key.n&.num_bits when :dsa - key.p.num_bits + key.p&.num_bits when :ecdsa - key.group.order.num_bits + key.group.order&.num_bits when :ed25519 256 else -- cgit v1.2.1 From 1fb4ed21553c3741dd96894d2489628713bf50b0 Mon Sep 17 00:00:00 2001 From: Jeff Stubler Date: Mon, 20 Mar 2017 08:34:47 -0500 Subject: Change coverage badge rounding for other CI system consistency --- lib/gitlab/badge/coverage/report.rb | 2 +- lib/gitlab/badge/coverage/template.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 9a0482306b7..778d78185ff 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -23,7 +23,7 @@ module Gitlab @coverage ||= raw_coverage return unless @coverage - @coverage.to_i + @coverage.to_f.round(2) end def metadata diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb index fcecb1d9665..afbf9dd17e3 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/badge/coverage/template.rb @@ -25,7 +25,7 @@ module Gitlab end def value_text - @status ? "#{@status}%" : 'unknown' + @status ? ("%.2f%%" % @status) : 'unknown' end def key_width @@ -33,7 +33,7 @@ module Gitlab end def value_width - @status ? 36 : 58 + @status ? 54 : 58 end def value_color -- cgit v1.2.1 From 948150f050d14811f6fe1b327387e3c2c1b1fe31 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 2 Feb 2018 11:23:05 +0100 Subject: Avoid error when no restricted levels are defined When no visibility levels are defined they could sometimes return `nil` instead of an empty array. In this case we want to allow all levels. --- lib/gitlab/visibility_level.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 6ced06a863d..0b757b2a646 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -60,7 +60,7 @@ module Gitlab def allowed_levels restricted_levels = current_application_settings.restricted_visibility_levels - self.values - restricted_levels + self.values - Array(restricted_levels) end def closest_allowed_level(target_level) -- cgit v1.2.1 From 74ddc80590053b04b90c35ae3e1f46bfbd9d0d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Mon, 29 Jan 2018 16:06:17 -0500 Subject: add the uploader context to the upload model --- lib/gitlab/gfm/uploads_rewriter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 3fdc3c27f73..1b74f735679 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -46,7 +46,7 @@ module Gitlab private def find_file(project, secret, file) - uploader = FileUploader.new(project, secret) + uploader = FileUploader.new(project, secret: secret) uploader.retrieve_from_store!(file) uploader.file end -- cgit v1.2.1 From eaada9d7066a20b5af815f723e09cde60a5c8c10 Mon Sep 17 00:00:00 2001 From: Mario de la Ossa Date: Fri, 2 Feb 2018 18:39:55 +0000 Subject: use Gitlab::UserSettings directly as a singleton instead of including/extending it --- lib/api/helpers/runner.rb | 6 +- lib/email_template_interceptor.rb | 4 +- lib/gitlab/asciidoc.rb | 8 +- lib/gitlab/auth.rb | 6 +- lib/gitlab/current_settings.rb | 108 +++++++++++---------- lib/gitlab/gon_helper.rb | 5 +- lib/gitlab/legacy_github_import/project_creator.rb | 4 +- lib/gitlab/metrics/prometheus.rb | 3 +- lib/gitlab/middleware/go.rb | 3 +- lib/gitlab/performance_bar.rb | 4 +- lib/gitlab/polling_interval.rb | 6 +- lib/gitlab/protocol_access.rb | 6 +- lib/gitlab/recaptcha.rb | 10 +- lib/gitlab/sentry.rb | 4 +- lib/gitlab/usage_data.rb | 8 +- lib/gitlab/visibility_level.rb | 5 +- 16 files changed, 85 insertions(+), 105 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 2cae53dba53..3d0d1287407 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,15 +1,13 @@ module API module Helpers module Runner - include Gitlab::CurrentSettings - JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 10 * 60 def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], - current_application_settings.runners_registration_token) + Gitlab::CurrentSettings.runners_registration_token) end def get_runner_version_from_params @@ -70,7 +68,7 @@ module API end def max_artifacts_size - current_application_settings.max_artifacts_size.megabytes.to_i + Gitlab::CurrentSettings.max_artifacts_size.megabytes.to_i end end end diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb index f2bf3d0fb2b..3978a6d9fe4 100644 --- a/lib/email_template_interceptor.rb +++ b/lib/email_template_interceptor.rb @@ -1,10 +1,8 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class EmailTemplateInterceptor - extend Gitlab::CurrentSettings - def self.delivering_email(message) # Remove HTML part if HTML emails are disabled. - unless current_application_settings.html_emails_enabled + unless Gitlab::CurrentSettings.html_emails_enabled message.parts.delete_if do |part| part.content_type.start_with?('text/html') end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index cead1c7eacd..ee7f4be6b9f 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -6,8 +6,6 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - extend Gitlab::CurrentSettings - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' @@ -33,9 +31,9 @@ module Gitlab def self.plantuml_setup Asciidoctor::PlantUml.configure do |conf| - conf.url = current_application_settings.plantuml_url - conf.svg_enable = current_application_settings.plantuml_enabled - conf.png_enable = current_application_settings.plantuml_enabled + conf.url = Gitlab::CurrentSettings.plantuml_url + conf.svg_enable = Gitlab::CurrentSettings.plantuml_enabled + conf.png_enable = Gitlab::CurrentSettings.plantuml_enabled conf.txt_enable = false end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 65d7fd3ec70..05932378173 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -14,8 +14,6 @@ module Gitlab DEFAULT_SCOPES = [:api].freeze class << self - include Gitlab::CurrentSettings - def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -57,7 +55,7 @@ module Gitlab if user.nil? || user.ldap_user? # Second chance - try LDAP authentication Gitlab::LDAP::Authentication.login(login, password) - elsif current_application_settings.password_authentication_enabled_for_git? + elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git? user if user.active? && user.valid_password?(password) end end @@ -87,7 +85,7 @@ module Gitlab private def authenticate_using_internal_or_ldap_password? - current_application_settings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? end def service_request_check(login, password, project) diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 91fd9cc7631..b7c596a973d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,73 +1,79 @@ module Gitlab module CurrentSettings - extend self + class << self + def current_application_settings + if RequestStore.active? + RequestStore.fetch(:current_application_settings) { ensure_application_settings! } + else + ensure_application_settings! + end + end - def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! + def fake_application_settings(defaults = ::ApplicationSetting.defaults) + Gitlab::FakeApplicationSettings.new(defaults) end - end - delegate :sidekiq_throttling_enabled?, to: :current_application_settings + def method_missing(name, *args, &block) + current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + end - def fake_application_settings(defaults = ::ApplicationSetting.defaults) - FakeApplicationSettings.new(defaults) - end + def respond_to_missing?(name, include_private = false) + current_application_settings.respond_to?(name, include_private) || super + end - private + private - def ensure_application_settings! - return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + def ensure_application_settings! + return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - cached_application_settings || uncached_application_settings - end + cached_application_settings || uncached_application_settings + end - def cached_application_settings - begin - ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL - # In case Redis isn't running or the Redis UNIX socket file is not available + def cached_application_settings + begin + ::ApplicationSetting.cached + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL + # In case Redis isn't running or the Redis UNIX socket file is not available + end end - end - def uncached_application_settings - return fake_application_settings unless connect_to_db? + def uncached_application_settings + return fake_application_settings unless connect_to_db? - db_settings = ::ApplicationSetting.current + db_settings = ::ApplicationSetting.current - # If there are pending migrations, it's possible there are columns that - # need to be added to the application settings. To prevent Rake tasks - # and other callers from failing, use any loaded settings and return - # defaults for missing columns. - if ActiveRecord::Migrator.needs_migration? - defaults = ::ApplicationSetting.defaults - defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? - return fake_application_settings(defaults) - end + # If there are pending migrations, it's possible there are columns that + # need to be added to the application settings. To prevent Rake tasks + # and other callers from failing, use any loaded settings and return + # defaults for missing columns. + if ActiveRecord::Migrator.needs_migration? + defaults = ::ApplicationSetting.defaults + defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? + return fake_application_settings(defaults) + end - return db_settings if db_settings.present? + return db_settings if db_settings.present? - ::ApplicationSetting.create_from_defaults || in_memory_application_settings - end + ::ApplicationSetting.create_from_defaults || in_memory_application_settings + end - def in_memory_application_settings - @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables - rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError - # In case migrations the application_settings table is not created yet, - # we fallback to a simple OpenStruct - fake_application_settings - end + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + fake_application_settings + end - def connect_to_db? - # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised - active_db_connection = ActiveRecord::Base.connection.active? rescue false + def connect_to_db? + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = ActiveRecord::Base.connection.active? rescue false - active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') - rescue ActiveRecord::NoDatabaseError - false + active_db_connection && + ActiveRecord::Base.connection.table_exists?('application_settings') + rescue ActiveRecord::NoDatabaseError + false + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9148d7571f2..86a90d57d9c 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,12 +3,11 @@ module Gitlab module GonHelper include WebpackHelper - include Gitlab::CurrentSettings def add_gon_variables gon.api_version = 'v4' gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.max_file_size = current_application_settings.max_attachment_size + gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root @@ -16,7 +15,7 @@ module Gitlab gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') - gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled + gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index 41e7eac4d08..cbabe5454ca 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -1,8 +1,6 @@ module Gitlab module LegacyGithubImport class ProjectCreator - include Gitlab::CurrentSettings - attr_reader :repo, :name, :namespace, :current_user, :session_data, :type def initialize(repo, name, namespace, current_user, session_data, type: 'github') @@ -36,7 +34,7 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.default_project_visibility end # diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index f07ea3560ff..d12ba0ec176 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -71,8 +71,7 @@ module Gitlab end def prometheus_metrics_enabled_unmemoized - metrics_folder_present? && - Gitlab::CurrentSettings.current_application_settings[:prometheus_metrics_enabled] || false + metrics_folder_present? && Gitlab::CurrentSettings.prometheus_metrics_enabled || false end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index afbc2600634..1a570f480c6 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -4,7 +4,6 @@ module Gitlab module Middleware class Go include ActionView::Helpers::TagHelper - include Gitlab::CurrentSettings PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze @@ -42,7 +41,7 @@ module Gitlab project_url = URI.join(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh' + repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh' shell = config.gitlab_shell port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index e29e168fc5a..6c2b2036074 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,7 +1,5 @@ module Gitlab module PerformanceBar - extend Gitlab::CurrentSettings - ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze EXPIRY_TIME = 5.minutes @@ -13,7 +11,7 @@ module Gitlab end def self.allowed_group_id - current_application_settings.performance_bar_allowed_group_id + Gitlab::CurrentSettings.performance_bar_allowed_group_id end def self.allowed_user_ids diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index 4780675a492..fe4bdfe3831 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -1,12 +1,10 @@ module Gitlab class PollingInterval - extend Gitlab::CurrentSettings - HEADER_NAME = 'Poll-Interval'.freeze def self.set_header(response, interval:) if polling_enabled? - multiplier = current_application_settings.polling_interval_multiplier + multiplier = Gitlab::CurrentSettings.polling_interval_multiplier value = (interval * multiplier).to_i else value = -1 @@ -16,7 +14,7 @@ module Gitlab end def self.polling_enabled? - !current_application_settings.polling_interval_multiplier.zero? + !Gitlab::CurrentSettings.polling_interval_multiplier.zero? end end end diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index 09fa14764e6..2819c7d062c 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -1,14 +1,12 @@ module Gitlab module ProtocolAccess - extend Gitlab::CurrentSettings - def self.allowed?(protocol) if protocol == 'web' true - elsif current_application_settings.enabled_git_access_protocol.blank? + elsif Gitlab::CurrentSettings.enabled_git_access_protocol.blank? true else - protocol == current_application_settings.enabled_git_access_protocol + protocol == Gitlab::CurrentSettings.enabled_git_access_protocol end end end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index c463dd487a0..c9efa28d7e7 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -1,12 +1,10 @@ module Gitlab module Recaptcha - extend Gitlab::CurrentSettings - def self.load_configurations! - if current_application_settings.recaptcha_enabled + if Gitlab::CurrentSettings.recaptcha_enabled ::Recaptcha.configure do |config| - config.public_key = current_application_settings.recaptcha_site_key - config.private_key = current_application_settings.recaptcha_private_key + config.public_key = Gitlab::CurrentSettings.recaptcha_site_key + config.private_key = Gitlab::CurrentSettings.recaptcha_private_key end true @@ -14,7 +12,7 @@ module Gitlab end def self.enabled? - current_application_settings.recaptcha_enabled + Gitlab::CurrentSettings.recaptcha_enabled end end end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 159d0e7952e..4a22fc80f75 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -1,9 +1,7 @@ module Gitlab module Sentry - extend Gitlab::CurrentSettings - def self.enabled? - Rails.env.production? && current_application_settings.sentry_enabled? + Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled? end def self.context(current_user = nil) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 2adcc9809b3..9d13d1d781f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,8 +1,6 @@ module Gitlab class UsageData class << self - include Gitlab::CurrentSettings - def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } end @@ -19,7 +17,7 @@ module Gitlab def license_usage_data usage_data = { - uuid: current_application_settings.uuid, + uuid: Gitlab::CurrentSettings.uuid, hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, active_user_count: User.active.count, @@ -79,9 +77,9 @@ module Gitlab def features_usage_data_ce { - signup: current_application_settings.allow_signup?, + signup: Gitlab::CurrentSettings.allow_signup?, ldap: Gitlab.config.ldap.enabled, - gravatar: current_application_settings.gravatar_enabled?, + gravatar: Gitlab::CurrentSettings.gravatar_enabled?, omniauth: Gitlab.config.omniauth.enabled, reply_by_email: Gitlab::IncomingEmail.enabled?, container_registry: Gitlab.config.registry.enabled, diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 6ced06a863d..a682de96b3b 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -5,7 +5,6 @@ # module Gitlab module VisibilityLevel - extend CurrentSettings extend ActiveSupport::Concern included do @@ -58,7 +57,7 @@ module Gitlab end def allowed_levels - restricted_levels = current_application_settings.restricted_visibility_levels + restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels self.values - restricted_levels end @@ -81,7 +80,7 @@ module Gitlab end def non_restricted_level?(level) - restricted_levels = current_application_settings.restricted_visibility_levels + restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels if restricted_levels.nil? true -- cgit v1.2.1 From 98affa75edde3969e0c119e9f2140710e654abf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Wed, 31 Jan 2018 16:25:38 -0300 Subject: Refactor Gitlab::Git code related to LFS changes for Gitaly migration We stop relying on Gitlab::Git::Env for the RevList class, and use Gitlab::Git::Repository#run_git methods inteaad. The refactor also fixes another issue, since we now top using "path_to_repo" (which is a Repository model method). --- lib/gitlab/checks/force_push.rb | 4 +-- lib/gitlab/git/lfs_changes.rb | 3 +- lib/gitlab/git/popen.rb | 2 +- lib/gitlab/git/repository.rb | 32 ++++++++++++++++++--- lib/gitlab/git/rev_list.rb | 63 ++++++++++++----------------------------- 5 files changed, 50 insertions(+), 54 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index dc5d285ea65..c9c3050cfc2 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -15,8 +15,8 @@ module Gitlab .ancestor?(oldrev, newrev) else Gitlab::Git::RevList.new( - path_to_repo: project.repository.path_to_repo, - oldrev: oldrev, newrev: newrev).missed_ref.present? + project.repository.raw, oldrev: oldrev, newrev: newrev + ).missed_ref.present? end end end diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index 732dd5d998a..48434047fce 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -25,8 +25,7 @@ module Gitlab private def rev_list - ::Gitlab::Git::RevList.new(path_to_repo: @repository.path_to_repo, - newrev: @newrev) + Gitlab::Git::RevList.new(@repository, newrev: @newrev) end end end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index e0bd2bbe47b..c1767046ff0 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -25,7 +25,7 @@ module Gitlab stdin.close if lazy_block - return lazy_block.call(stdout.lazy) + return [lazy_block.call(stdout.lazy), 0] else cmd_output << stdout.read end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7127f7858ee..635c94d9dd2 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1431,6 +1431,26 @@ module Gitlab end end + def rev_list(including: [], excluding: [], objects: false, &block) + args = ['rev-list'] + + args.push(*rev_list_param(including)) + + exclude_param = *rev_list_param(excluding) + if exclude_param.any? + args.push('--not') + args.push(*exclude_param) + end + + args.push('--objects') if objects + + run_git!(args, lazy_block: block) + end + + def missed_ref(oldrev, newrev) + run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) + end + private def local_write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -1460,7 +1480,7 @@ module Gitlab Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" end - def run_git(args, chdir: path, env: {}, nice: false, &block) + def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) cmd = [Gitlab.config.git.bin_path, *args] cmd.unshift("nice") if nice @@ -1470,12 +1490,12 @@ module Gitlab end circuit_breaker.perform do - popen(cmd, chdir, env, &block) + popen(cmd, chdir, env, lazy_block: lazy_block, &block) end end - def run_git!(args, chdir: path, env: {}, nice: false, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) + def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) raise GitError, output unless status.zero? @@ -2336,6 +2356,10 @@ module Gitlab rescue Rugged::ReferenceError 0 end + + def rev_list_param(spec) + spec == :all ? ['--all'] : spec + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index f8b2e7e0e21..38c3a55f96f 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -5,17 +5,17 @@ module Gitlab class RevList include Gitlab::Git::Popen - attr_reader :oldrev, :newrev, :path_to_repo + attr_reader :oldrev, :newrev, :repository - def initialize(path_to_repo:, newrev:, oldrev: nil) + def initialize(repository, newrev:, oldrev: nil) @oldrev = oldrev @newrev = newrev - @path_to_repo = path_to_repo + @repository = repository end # This method returns an array of new commit references def new_refs - execute([*base_args, newrev, '--not', '--all']) + repository.rev_list(including: newrev, excluding: :all).split("\n") end # Finds newly added objects @@ -28,66 +28,39 @@ module Gitlab # When given a block it will yield objects as a lazy enumerator so # the caller can limit work done instead of processing megabytes of data def new_objects(require_path: nil, not_in: nil, &lazy_block) - args = [*base_args, newrev, *not_in_refs(not_in), '--objects'] + opts = { + including: newrev, + excluding: not_in.nil? ? :all : not_in, + require_path: require_path + } - get_objects(args, require_path: require_path, &lazy_block) + get_objects(opts, &lazy_block) end def all_objects(require_path: nil, &lazy_block) - args = [*base_args, '--all', '--objects'] - - get_objects(args, require_path: require_path, &lazy_block) + get_objects(including: :all, require_path: require_path, &lazy_block) end # This methods returns an array of missed references # # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref - execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) + repository.missed_ref(oldrev, newrev).split("\n") end private - def not_in_refs(references) - return ['--not', '--all'] unless references - return [] if references.empty? - - references.prepend('--not') - end - def execute(args) - output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash) - - unless status.zero? - raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" - end - - output.split("\n") - end - - def lazy_execute(args, &lazy_block) - popen(args, nil, Gitlab::Git::Env.to_env_hash, lazy_block: lazy_block) - end - - def base_args - [ - Gitlab.config.git.bin_path, - "--git-dir=#{path_to_repo}", - 'rev-list' - ] + repository.rev_list(args).split("\n") end - def get_objects(args, require_path: nil) - if block_given? - lazy_execute(args) do |lazy_output| - objects = objects_from_output(lazy_output, require_path: require_path) + def get_objects(including: [], excluding: [], require_path: nil) + opts = { including: including, excluding: excluding, objects: true } - yield(objects) - end - else - object_output = execute(args) + repository.rev_list(opts) do |lazy_output| + objects = objects_from_output(lazy_output, require_path: require_path) - objects_from_output(object_output, require_path: require_path) + yield(objects) end end -- cgit v1.2.1 From fac2047c091a9dfb08e98aea17b0b6bf6b124969 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 2 Feb 2018 15:13:32 -0600 Subject: Use %r around multi-line regular expressions --- lib/banzai/color_parser.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb index 0cd9085190c..355c364b07b 100644 --- a/lib/banzai/color_parser.rb +++ b/lib/banzai/color_parser.rb @@ -7,7 +7,7 @@ module Banzai DEGS = /-?\d+(?:deg)?/i # [-]digits[deg] RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/ - RGB_FORMAT = / + RGB_FORMAT = %r{ (?:rgba? \( (?: @@ -18,15 +18,15 @@ module Banzai #{ALPHA_CHANNEL} \) ) - /xi - HSL_FORMAT = / + }xi + HSL_FORMAT = %r{ (?:hsla? \( (?:#{DEGS}|#{RADS}),\s*#{PERCENTS},\s*#{PERCENTS} #{ALPHA_CHANNEL} \) ) - /xi + }xi FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze -- cgit v1.2.1 From e60bf2f256746e49eb79b6cd97dfe2147ea957b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Fri, 2 Feb 2018 18:30:03 -0300 Subject: Incorporate OperationService.UserSquash Gitaly RPC --- lib/gitlab/git/repository.rb | 63 ++++++++++++++++----------- lib/gitlab/gitaly_client/operation_service.rb | 26 +++++++++++ 2 files changed, 63 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f28624ff37a..359ad9f4521 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1222,33 +1222,13 @@ module Gitlab end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) - squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) - env = git_env_for_user(user).merge( - 'GIT_AUTHOR_NAME' => author.name, - 'GIT_AUTHOR_EMAIL' => author.email - ) - diff_range = "#{start_sha}...#{end_sha}" - diff_files = run_git!( - %W(diff --name-only --diff-filter=a --binary #{diff_range}) - ).chomp - - with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do - # Apply diff of the `diff_range` to the worktree - diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| - stdin.write(diff) + gitaly_migrate(:squash) do |is_enabled| + if is_enabled + gitaly_operation_client.user_squash(user, squash_id, branch, + start_sha, end_sha, author, message) + else + git_squash(user, squash_id, branch, start_sha, end_sha, author, message) end - - # Commit the `diff_range` diff - run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) - - # Return the squash sha. May print a warning for ambiguous refs, but - # we can ignore that with `--quiet` and just take the SHA, if present. - # HEAD here always refers to the current HEAD commit, even if there is - # another ref called HEAD. - run_git!( - %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env - ).chomp end end @@ -2164,6 +2144,37 @@ module Gitlab end end + def git_squash(user, squash_id, branch, start_sha, end_sha, author, message) + squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) + env = git_env_for_user(user).merge( + 'GIT_AUTHOR_NAME' => author.name, + 'GIT_AUTHOR_EMAIL' => author.email + ) + diff_range = "#{start_sha}...#{end_sha}" + diff_files = run_git!( + %W(diff --name-only --diff-filter=a --binary #{diff_range}) + ).chomp + + with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do + # Apply diff of the `diff_range` to the worktree + diff = run_git!(%W(diff --binary #{diff_range})) + run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + stdin.write(diff) + end + + # Commit the `diff_range` diff + run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + + # Return the squash sha. May print a warning for ambiguous refs, but + # we can ignore that with `--quiet` and just take the SHA, if present. + # HEAD here always refers to the current HEAD commit, even if there is + # another ref called HEAD. + run_git!( + %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env + ).chomp + end + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index cd2734b5a07..831cfd1e014 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -183,6 +183,32 @@ module Gitlab end end + def user_squash(user, squash_id, branch, start_sha, end_sha, author, message) + request = Gitaly::UserSquashRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + squash_id: squash_id.to_s, + branch: encode_binary(branch), + start_sha: start_sha, + end_sha: end_sha, + author: Gitlab::Git::User.from_gitlab(author).to_gitaly, + commit_message: encode_binary(message) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_squash, + request + ) + + if response.git_error.presence + raise Gitlab::Git::Repository::GitError, response.git_error + end + + response.squash_sha + end + def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, start_branch_name, start_repository) -- cgit v1.2.1 From 38c242126d10d0887ca0d3dd8565ef2669dfec46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Sun, 4 Feb 2018 18:34:21 +0100 Subject: Refactor runner attribute caching implementation --- lib/api/helpers/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 8f45cae0e60..87ba40e26e1 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -20,7 +20,7 @@ module API def authenticate_runner! forbidden! unless current_runner - current_runner.update_runner_info(get_runner_version_from_params) + current_runner.update_cached_info(get_runner_version_from_params) end def current_runner -- cgit v1.2.1 From bd132dfe72ba6c083e696dfc20ce895f6e5021c3 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Mon, 5 Feb 2018 17:44:23 +0800 Subject: Make find_user_from_sources extensible for EE --- lib/api/api_guard.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 9aeebc34525..c2113551207 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -42,7 +42,7 @@ module API include Gitlab::Auth::UserAuthFinders def find_current_user! - user = find_user_from_access_token || find_user_from_warden + user = find_user_from_sources return unless user forbidden!('User is blocked') unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) @@ -50,6 +50,10 @@ module API user end + def find_user_from_sources + find_user_from_access_token || find_user_from_warden + end + private # An array of scopes that were registered (using `allow_access_with_scope`) -- cgit v1.2.1 From be26c4e728a2704d79a17df0edf84689b44eebd6 Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Mon, 5 Feb 2018 12:26:25 +0100 Subject: Encode GitalyClient::WikiPage attributes to UTF-8 --- lib/gitlab/gitaly_client/wiki_page.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index 7339468e911..a02d15db5dd 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -4,6 +4,7 @@ module Gitlab ATTRS = %i(title format url_path path name historical raw_data).freeze include AttributesBag + include Gitlab::EncodingHelper def initialize(params) super @@ -11,6 +12,10 @@ module Gitlab # All gRPC strings in a response are frozen, so we get an unfrozen # version here so appending to `raw_data` doesn't blow up. @raw_data = @raw_data.dup + + @title = encode_utf8(@title) + @path = encode_utf8(@path) + @name = encode_utf8(@name) end def historical? -- cgit v1.2.1 From 1f912880dbf6337718611edbd9c2fb52c5639476 Mon Sep 17 00:00:00 2001 From: Joshua Lambert Date: Mon, 5 Feb 2018 12:46:31 +0000 Subject: Enable Prometheus metrics for deployed Ingresses --- lib/gitlab/kubernetes/helm/pod.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index a3216759cae..ca5e06009fa 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -64,7 +64,7 @@ module Gitlab { name: 'configuration-volume', configMap: { - name: 'values-content-configuration', + name: "values-content-configuration-#{command.name}", items: [{ key: 'values', path: 'values.yaml' }] } } @@ -81,7 +81,11 @@ module Gitlab def create_config_map resource = ::Kubeclient::Resource.new - resource.metadata = { name: 'values-content-configuration', namespace: namespace_name, labels: { name: 'values-content-configuration' } } + resource.metadata = { + name: "values-content-configuration-#{command.name}", + namespace: namespace_name, + labels: { name: "values-content-configuration-#{command.name}" } + } resource.data = { values: File.read(command.chart_values_file) } kubeclient.create_config_map(resource) end -- cgit v1.2.1 From 27c08a16891193227fc83bf4692b4d31a3b2539f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Mon, 5 Feb 2018 17:17:21 +0000 Subject: Allow moving wiki pages from the UI --- lib/gitlab/git/wiki.rb | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index ccdb8975342..daa17fc72cf 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -25,8 +25,9 @@ module Gitlab @repository.exists? end + # Disabled because of https://gitlab.com/gitlab-org/gitaly/merge_requests/539 def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_write_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_write_page(name, format, content, commit_details) gollum_wiki.clear_cache @@ -47,8 +48,9 @@ module Gitlab end end + # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42094 def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_update_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_update_page(page_path, title, format, content, commit_details) gollum_wiki.clear_cache @@ -68,8 +70,9 @@ module Gitlab end end + # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42039 def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_find_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_find_page(title: title, version: version, dir: dir) else @@ -192,7 +195,10 @@ module Gitlab assert_type!(format, Symbol) assert_type!(commit_details, CommitDetails) - gollum_wiki.write_page(name, format, content, commit_details.to_h) + filename = File.basename(name) + dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir + + gollum_wiki.write_page(filename, format, content, commit_details.to_h, dir) nil rescue Gollum::DuplicatePageError => e @@ -210,7 +216,15 @@ module Gitlab assert_type!(format, Symbol) assert_type!(commit_details, CommitDetails) - gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + page = gollum_page_by_path(page_path) + committer = Gollum::Committer.new(page.wiki, commit_details.to_h) + + # Instead of performing two renames if the title has changed, + # the update_page will only update the format and content and + # the rename_page will do anything related to moving/renaming + gollum_wiki.update_page(page, page.name, format, content, committer: committer) + gollum_wiki.rename_page(page, title, committer: committer) + committer.commit nil end -- cgit v1.2.1 From cd461400eb2e592c52c1b6ba61771df2fa2913bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Mon, 5 Feb 2018 22:35:34 +0000 Subject: Added ldap config setting to lower case usernames --- lib/gitlab/ldap/auth_hash.rb | 6 ++++++ lib/gitlab/ldap/config.rb | 4 ++++ lib/gitlab/ldap/person.rb | 4 +++- 3 files changed, 13 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 1bd0965679a..96171dc26c4 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -7,6 +7,12 @@ module Gitlab @uid ||= Gitlab::LDAP::Person.normalize_dn(super) end + def username + super.tap do |username| + username.downcase! if ldap_config.lowercase_usernames + end + end + private def get_info(key) diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index cde60addcf7..47b3fce3e7a 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -139,6 +139,10 @@ module Gitlab options['allow_username_or_email_login'] end + def lowercase_usernames + options['lowercase_usernames'] + end + def name_proc if allow_username_or_email_login proc { |name| name.gsub(/@.*\z/, '') } diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index e81cec6ba1a..b91757c2a4b 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -82,7 +82,9 @@ module Gitlab # be returned. We need only one for username. # Ex. `uid` returns only one value but `mail` may # return an array of multiple email addresses. - [username].flatten.first + [username].flatten.first.tap do |username| + username.downcase! if config.lowercase_usernames + end end def email -- cgit v1.2.1 From 63a9d582aa88d774af5eff124b693df6271ae7bc Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 25 Jan 2018 18:50:56 +0900 Subject: Trace as artifacts --- lib/api/runner.rb | 11 ++++++++++- lib/gitlab/ci/trace.rb | 22 ++++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 1f80646a2ea..bab9d263e8d 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -120,7 +120,16 @@ module API put '/:id' do job = authenticate_job! - job.trace.set(params[:trace]) if params[:trace] + if params[:trace] + # Overwrite live-trace by full-trace + job.trace.set(params[:trace]) + + # Move full-trace to JobArtifactUploader#default_path + job.build_job_artifacts_trace( + project: job.project, + file_type: :trace, + file: UploadedFile.new(job.trace.current_path, 'trace.log')) + end Gitlab::Metrics.add_event(:update_build, project: job.project.full_path) diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index baf55b1fa07..a6827d7a38a 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -52,12 +52,14 @@ module Gitlab end def exist? - current_path.present? || old_trace.present? + trace_artifact&.exists? || current_path.present? || old_trace.present? end def read stream = Gitlab::Ci::Trace::Stream.new do - if current_path + if trace_artifact + trace_artifact.open + elsif current_path File.open(current_path, "rb") elsif old_trace StringIO.new(old_trace) @@ -104,12 +106,6 @@ module Gitlab end end - def current_path - @current_path ||= paths.find do |trace_path| - File.exist?(trace_path) - end - end - def paths [ default_path, @@ -117,6 +113,12 @@ module Gitlab ].compact end + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + def default_directory File.join( Settings.gitlab_ci.builds_path, @@ -137,6 +139,10 @@ module Gitlab "#{job.id}.log" ) if job.project&.ci_id end + + def trace_artifact + job.job_artifacts_trace + end end end end -- cgit v1.2.1 From 8a89efd87a137634381a93c45c05e587eb207b94 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 25 Jan 2018 22:46:38 +0900 Subject: Remove unnecessary changes --- lib/gitlab/ci/trace.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index a6827d7a38a..d7d4b02b516 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -84,6 +84,8 @@ module Gitlab end def erase! + job.job_artifacts_trace&.destory + paths.each do |trace_path| FileUtils.rm(trace_path, force: true) end @@ -106,6 +108,14 @@ module Gitlab end end + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + + ## + # This doesn't include the latest path JobArtifactUploader#default_path. def paths [ default_path, @@ -113,12 +123,6 @@ module Gitlab ].compact end - def current_path - @current_path ||= paths.find do |trace_path| - File.exist?(trace_path) - end - end - def default_directory File.join( Settings.gitlab_ci.builds_path, -- cgit v1.2.1 From 1d983024db59ab9791c9a8705e71f20cd9346893 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 25 Jan 2018 23:46:00 +0900 Subject: Move default_path to legacy_default_path. Switch to the new path for live-trace --- lib/gitlab/ci/trace.rb | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index d7d4b02b516..aff60467bb1 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,3 +1,11 @@ +## +# Current status of paths +# Era 1: Live/Full traces in database (ci_builds.trace) +# Era 2: Live/Full traces in `setting_root/YYYY_MM/project_ci_id/job_id.log` +# Era 3: Live/Full traces in `setting_root/YYYY_MM/project_id/job_id.log` +# Era 4: Live traces in `setting_root/live_trace/job_id.log`. Full traces in JobArtifactUploader#legacy_default_path. +# +# The legacy paths are to be migrated to the latest era. module Gitlab module Ci class Trace @@ -99,12 +107,12 @@ module Gitlab return current_path if current_path ensure_directory - default_path + live_trace_default_path end def ensure_directory - unless Dir.exist?(default_directory) - FileUtils.mkdir_p(default_directory) + unless Dir.exist?(live_trace_default_directory) + FileUtils.mkdir_p(live_trace_default_directory) end end @@ -115,24 +123,34 @@ module Gitlab end ## - # This doesn't include the latest path JobArtifactUploader#default_path. + # This method doesn't include the latest path, which is JobArtifactUploader#default_path, + # Because, in EE, traces can be moved to ObjectStorage, so checking paths in Filestorage doesn't make sense. + # All legacy paths (`legacy_default_path` and `deprecated_path`) are to be migrated to JobArtifactUploader#default_path def paths [ - default_path, + live_trace_default_path, + legacy_default_path, deprecated_path ].compact end - def default_directory + def live_trace_default_directory File.join( Settings.gitlab_ci.builds_path, - job.created_at.utc.strftime("%Y_%m"), - job.project_id.to_s + 'live_trace' ) end - def default_path - File.join(default_directory, "#{job.id}.log") + def live_trace_default_path + File.join(live_trace_default_directory, "#{job.id}.log") + end + + def legacy_default_path + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project_id.to_s, + "#{job.id}.log") end def deprecated_path -- cgit v1.2.1 From b9692501060b3543bd3289bd806920044516db8c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 25 Jan 2018 23:52:35 +0900 Subject: Check existance of the file when trace_artifact is read --- lib/gitlab/ci/trace.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index aff60467bb1..27f5869770e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -65,7 +65,7 @@ module Gitlab def read stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact + if trace_artifact&.exists? trace_artifact.open elsif current_path File.open(current_path, "rb") @@ -92,7 +92,7 @@ module Gitlab end def erase! - job.job_artifacts_trace&.destory + trace_artifact&.destory paths.each do |trace_path| FileUtils.rm(trace_path, force: true) -- cgit v1.2.1 From 002f314f320c5731681297225fff5b528de88ed2 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 26 Jan 2018 02:21:31 +0900 Subject: Expose current_path --- lib/gitlab/ci/trace.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 27f5869770e..307e3bf1c8f 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -101,6 +101,12 @@ module Gitlab job.erase_old_trace! end + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + private def ensure_path @@ -116,12 +122,6 @@ module Gitlab end end - def current_path - @current_path ||= paths.find do |trace_path| - File.exist?(trace_path) - end - end - ## # This method doesn't include the latest path, which is JobArtifactUploader#default_path, # Because, in EE, traces can be moved to ObjectStorage, so checking paths in Filestorage doesn't make sense. -- cgit v1.2.1 From 5f6d826165aa975735cd543dba2b91999c115545 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 27 Jan 2018 02:00:29 +0900 Subject: Add CreateArtifactsTraceWorker --- lib/api/runner.rb | 11 +---------- lib/gitlab/ci/trace.rb | 12 ++++++------ 2 files changed, 7 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/api/runner.rb b/lib/api/runner.rb index bab9d263e8d..1f80646a2ea 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -120,16 +120,7 @@ module API put '/:id' do job = authenticate_job! - if params[:trace] - # Overwrite live-trace by full-trace - job.trace.set(params[:trace]) - - # Move full-trace to JobArtifactUploader#default_path - job.build_job_artifacts_trace( - project: job.project, - file_type: :trace, - file: UploadedFile.new(job.trace.current_path, 'trace.log')) - end + job.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: job.project.full_path) diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 307e3bf1c8f..27f5869770e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -101,12 +101,6 @@ module Gitlab job.erase_old_trace! end - def current_path - @current_path ||= paths.find do |trace_path| - File.exist?(trace_path) - end - end - private def ensure_path @@ -122,6 +116,12 @@ module Gitlab end end + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + ## # This method doesn't include the latest path, which is JobArtifactUploader#default_path, # Because, in EE, traces can be moved to ObjectStorage, so checking paths in Filestorage doesn't make sense. -- cgit v1.2.1 From 62e257f53ac801b5f30c84f3cb8ed359d7350826 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 27 Jan 2018 02:38:54 +0900 Subject: Revert live-trace path --- lib/gitlab/ci/trace.rb | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 27f5869770e..3d093a7f135 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -3,7 +3,7 @@ # Era 1: Live/Full traces in database (ci_builds.trace) # Era 2: Live/Full traces in `setting_root/YYYY_MM/project_ci_id/job_id.log` # Era 3: Live/Full traces in `setting_root/YYYY_MM/project_id/job_id.log` -# Era 4: Live traces in `setting_root/live_trace/job_id.log`. Full traces in JobArtifactUploader#legacy_default_path. +# Era 4: Live traces in `setting_root/YYYY_MM/project_id/job_id.log`. Full traces in JobArtifactUploader#default_path(FileStorage/ObjectStorage). # # The legacy paths are to be migrated to the latest era. module Gitlab @@ -65,7 +65,7 @@ module Gitlab def read stream = Gitlab::Ci::Trace::Stream.new do - if trace_artifact&.exists? + if trace_artifact trace_artifact.open elsif current_path File.open(current_path, "rb") @@ -92,7 +92,7 @@ module Gitlab end def erase! - trace_artifact&.destory + trace_artifact&.destroy paths.each do |trace_path| FileUtils.rm(trace_path, force: true) @@ -107,12 +107,12 @@ module Gitlab return current_path if current_path ensure_directory - live_trace_default_path + default_path end def ensure_directory - unless Dir.exist?(live_trace_default_directory) - FileUtils.mkdir_p(live_trace_default_directory) + unless Dir.exist?(default_directory) + FileUtils.mkdir_p(default_directory) end end @@ -122,35 +122,23 @@ module Gitlab end end - ## - # This method doesn't include the latest path, which is JobArtifactUploader#default_path, - # Because, in EE, traces can be moved to ObjectStorage, so checking paths in Filestorage doesn't make sense. - # All legacy paths (`legacy_default_path` and `deprecated_path`) are to be migrated to JobArtifactUploader#default_path def paths [ - live_trace_default_path, - legacy_default_path, + default_path, deprecated_path ].compact end - def live_trace_default_directory + def default_directory File.join( Settings.gitlab_ci.builds_path, - 'live_trace' + job.created_at.utc.strftime("%Y_%m"), + job.project_id.to_s ) end - def live_trace_default_path - File.join(live_trace_default_directory, "#{job.id}.log") - end - - def legacy_default_path - File.join( - Settings.gitlab_ci.builds_path, - job.created_at.utc.strftime("%Y_%m"), - job.project_id.to_s, - "#{job.id}.log") + def default_path + File.join(default_directory, "#{job.id}.log") end def deprecated_path -- cgit v1.2.1 From abc64da90cc4341607b48c09b0920296b5fb9663 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 29 Jan 2018 00:03:22 +0900 Subject: Rename CreateArtifactsTraceService to CreateTraceArtifactService --- lib/gitlab/ci/trace.rb | 8 -------- 1 file changed, 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 3d093a7f135..f2e5124c8a8 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,11 +1,3 @@ -## -# Current status of paths -# Era 1: Live/Full traces in database (ci_builds.trace) -# Era 2: Live/Full traces in `setting_root/YYYY_MM/project_ci_id/job_id.log` -# Era 3: Live/Full traces in `setting_root/YYYY_MM/project_id/job_id.log` -# Era 4: Live traces in `setting_root/YYYY_MM/project_id/job_id.log`. Full traces in JobArtifactUploader#default_path(FileStorage/ObjectStorage). -# -# The legacy paths are to be migrated to the latest era. module Gitlab module Ci class Trace -- cgit v1.2.1 From 921d2afc6989dfa8220032984f657210c07e8792 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 18 Jan 2018 09:31:00 +0000 Subject: Adds option to push over HTTP to create a new project --- lib/gitlab/git_access.rb | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 56f6febe86d..9427a5e4baa 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -18,21 +18,23 @@ module Gitlab upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', read_only: 'The repository is temporarily read-only. Please try again later.', - cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance.", + create: "Creating a repository to that namespace is not allowed." }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :redirected_path + attr_reader :actor, :project, :protocol, :authentication_abilities, :redirected_path, :target_namespace - def initialize(actor, project, protocol, authentication_abilities:, redirected_path: nil) + def initialize(actor, project, protocol, authentication_abilities:, redirected_path: nil, target_namespace: nil) @actor = actor @project = project @protocol = protocol @redirected_path = redirected_path @authentication_abilities = authentication_abilities + @target_namespace = target_namespace end def check(cmd, changes) @@ -44,6 +46,7 @@ module Gitlab check_command_disabled!(cmd) check_command_existence!(cmd) check_repository_existence! + check_repository_creation! case cmd when *DOWNLOAD_COMMANDS @@ -96,7 +99,7 @@ module Gitlab end def check_project_accessibility! - if project.blank? || !can_read_project? + if (project.blank? || !can_read_project?) && !can_create_project_in_namespace? raise NotFoundError, ERROR_MESSAGES[:project_not_found] end end @@ -140,11 +143,19 @@ module Gitlab end def check_repository_existence! - unless project.repository.exists? + if (project.blank? || !project.repository.exists?) && !can_create_project_in_namespace? raise UnauthorizedError, ERROR_MESSAGES[:no_repo] end end + def check_repository_creation! + return unless target_namespace + + unless can_create_project_in_namespace? + raise UnauthorizedError, ERROR_MESSAGES[:create] + end + end + def check_download_access! return if deploy_key? @@ -158,6 +169,8 @@ module Gitlab end def check_push_access!(changes) + return if can_create_project_in_namespace? + if project.repository_read_only? raise UnauthorizedError, ERROR_MESSAGES[:read_only] end @@ -234,6 +247,12 @@ module Gitlab end || Guest.can?(:read_project, project) end + def can_create_project_in_namespace? + return unless target_namespace + + actor.can?(:create_projects, target_namespace) + end + def http? protocol == 'http' end -- cgit v1.2.1 From 35882e681b681f68a818bda9a8d2624edfecc219 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 18 Jan 2018 16:07:07 +0000 Subject: Adds option to push over SSH to create a new project --- lib/api/internal.rb | 15 ++++++++++++++- lib/gitlab/git_access.rb | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 063f0d6599c..a83f714a1f3 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -43,7 +43,7 @@ module API access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess access_checker = access_checker_klass - .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path) + .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path, target_namespace: user.namespace) begin access_checker.check(params[:action], params[:changes]) @@ -51,6 +51,19 @@ module API return { status: false, message: e.message } end + if project.blank? && params[:action] == 'git-receive-pack' + project_params = { + description: "", + path: params[:project].split('/').last.gsub("\.git", ''), + namespace_id: user.namespace.id.to_s, + visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s + } + + @project = ::Projects::CreateService.new(user, project_params).execute + + return { status: false, message: "Could not create project" } unless @project.saved? + end + log_user_activity(actor) { diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 9427a5e4baa..7de8a99f9dc 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -250,7 +250,7 @@ module Gitlab def can_create_project_in_namespace? return unless target_namespace - actor.can?(:create_projects, target_namespace) + user.can?(:create_projects, target_namespace) end def http? -- cgit v1.2.1 From 32b2ff26011a5274bdb8a3dd41ad360a67c3148a Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Fri, 19 Jan 2018 13:04:14 +0000 Subject: Adds remote messsage when project is created in a push over SSH or HTTP --- lib/api/helpers/internal_helpers.rb | 16 ++++++++++ lib/api/internal.rb | 17 +++++++---- lib/gitlab/checks/new_project.rb | 60 +++++++++++++++++++++++++++++++++++++ lib/gitlab/git_access.rb | 35 ++++++++++++---------- lib/gitlab/git_access_wiki.rb | 4 +++ 5 files changed, 112 insertions(+), 20 deletions(-) create mode 100644 lib/gitlab/checks/new_project.rb (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index eb67de81a0d..c0fcae43638 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -29,6 +29,10 @@ module API {} end + def receive_pack? + params[:action] == 'git-receive-pack' + end + def fix_git_env_repository_paths(env, repository_path) if obj_dir_relative = env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence env['GIT_OBJECT_DIRECTORY'] = File.join(repository_path, obj_dir_relative) @@ -62,6 +66,18 @@ module API private + def project_path_regex + @project_regex ||= /\A(?#{Gitlab::PathRegex.full_namespace_route_regex})\/(?#{Gitlab::PathRegex.project_git_route_regex})\z/.freeze + end + + def project_match + @match ||= params[:project].match(project_path_regex).captures + end + + def namespace + @namespace ||= Namespace.find_by_path_or_name(project_match[:namespace_id]) + end + # rubocop:disable Gitlab/ModuleWithInstanceVariables def set_project if params[:gl_repository] diff --git a/lib/api/internal.rb b/lib/api/internal.rb index a83f714a1f3..f641ef457a3 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -43,7 +43,7 @@ module API access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess access_checker = access_checker_klass - .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path, target_namespace: user.namespace) + .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path, target_namespace: namespace) begin access_checker.check(params[:action], params[:changes]) @@ -51,17 +51,21 @@ module API return { status: false, message: e.message } end - if project.blank? && params[:action] == 'git-receive-pack' + if user && project.blank? && receive_pack? project_params = { description: "", - path: params[:project].split('/').last.gsub("\.git", ''), - namespace_id: user.namespace.id.to_s, + path: Project.parse_project_id(project_match[:project_name]), + namespace_id: namespace&.id, visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s } @project = ::Projects::CreateService.new(user, project_params).execute - return { status: false, message: "Could not create project" } unless @project.saved? + if @project.saved? + Gitlab::Checks::NewProject.new(user, @project, protocol).add_new_project_message + else + return { status: false, message: "Could not create project" } + end end log_user_activity(actor) @@ -221,7 +225,10 @@ module API # key could be used if user redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) + new_project_message = Gitlab::Checks::NewProject.fetch_new_project_message(user.id, project.id) + output[:redirected_message] = redirect_message if redirect_message + output[:new_project_message] = new_project_message if new_project_message end output diff --git a/lib/gitlab/checks/new_project.rb b/lib/gitlab/checks/new_project.rb new file mode 100644 index 00000000000..40d0acefaba --- /dev/null +++ b/lib/gitlab/checks/new_project.rb @@ -0,0 +1,60 @@ +module Gitlab + module Checks + class NewProject + NEW_PROJECT = "new_project".freeze + + def initialize(user, project, protocol) + @user = user + @project = project + @protocol = protocol + end + + def self.fetch_new_project_message(user_id, project_id) + new_project_key = new_project_message_key(user_id, project_id) + + Gitlab::Redis::SharedState.with do |redis| + message = redis.get(new_project_key) + redis.del(new_project_key) + message + end + end + + def add_new_project_message + Gitlab::Redis::SharedState.with do |redis| + key = self.class.new_project_message_key(user.id, project.id) + redis.setex(key, 5.minutes, new_project_message) + end + end + + def new_project_message + <<~MESSAGE.strip_heredoc + + The private project #{project.full_path} was created. + + To configure the remote, run: + git remote add origin #{git_url} + + To view the project, visit: + #{project_url} + + MESSAGE + end + + private + + attr_reader :project, :user, :protocol + + def self.new_project_message_key(user_id, project_id) + "#{NEW_PROJECT}:#{user_id}:#{project_id}" + end + + def project_url + Gitlab::Routing.url_helpers.project_url(project) + end + + def git_url + protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 7de8a99f9dc..598506aa418 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -13,6 +13,7 @@ module Gitlab 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', + namespace_not_found: 'The namespace you were looking for could not be found.', account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', @@ -41,18 +42,18 @@ module Gitlab check_protocol! check_valid_actor! check_active_user! - check_project_accessibility! + check_project_accessibility!(cmd) check_project_moved! check_command_disabled!(cmd) check_command_existence!(cmd) - check_repository_existence! - check_repository_creation! + check_repository_existence!(cmd) case cmd when *DOWNLOAD_COMMANDS check_download_access! when *PUSH_COMMANDS - check_push_access!(changes) + check_push_access!(cmd, changes) + check_repository_creation!(cmd) end true @@ -98,8 +99,8 @@ module Gitlab end end - def check_project_accessibility! - if (project.blank? || !can_read_project?) && !can_create_project_in_namespace? + def check_project_accessibility!(cmd) + if (project.blank? || !can_read_project?) && !can_create_project_in_namespace?(cmd) raise NotFoundError, ERROR_MESSAGES[:project_not_found] end end @@ -142,16 +143,20 @@ module Gitlab end end - def check_repository_existence! - if (project.blank? || !project.repository.exists?) && !can_create_project_in_namespace? + def check_repository_existence!(cmd) + if (project.blank? || !project.repository.exists?) && !can_create_project_in_namespace?(cmd) raise UnauthorizedError, ERROR_MESSAGES[:no_repo] end end - def check_repository_creation! - return unless target_namespace + def check_repository_creation!(cmd) + return unless project.blank? - unless can_create_project_in_namespace? + unless target_namespace + raise NotFoundError, ERROR_MESSAGES[:namespace_not_found] + end + + unless can_create_project_in_namespace?(cmd) raise UnauthorizedError, ERROR_MESSAGES[:create] end end @@ -168,8 +173,8 @@ module Gitlab end end - def check_push_access!(changes) - return if can_create_project_in_namespace? + def check_push_access!(cmd, changes) + return if project.blank? && can_create_project_in_namespace?(cmd) if project.repository_read_only? raise UnauthorizedError, ERROR_MESSAGES[:read_only] @@ -247,8 +252,8 @@ module Gitlab end || Guest.can?(:read_project, project) end - def can_create_project_in_namespace? - return unless target_namespace + def can_create_project_in_namespace?(cmd) + return false unless PUSH_COMMANDS.include?(cmd) && target_namespace user.can?(:create_projects, target_namespace) end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 1c9477e84b2..f679b5e8ed6 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -25,6 +25,10 @@ module Gitlab true end + def check_repository_creation!(cmd) + # Method not used in wiki + end + def push_to_read_only_message ERROR_MESSAGES[:read_only] end -- cgit v1.2.1 From bc78ae6985ee37f9ac2ffc2dbf6f445078d16038 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Mon, 22 Jan 2018 18:10:56 +0000 Subject: Add specs --- lib/api/helpers/internal_helpers.rb | 17 ++++++++++++++--- lib/api/internal.rb | 9 +-------- lib/gitlab/checks/new_project.rb | 2 ++ lib/gitlab/git_access.rb | 11 +++-------- 4 files changed, 20 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index c0fcae43638..fd568c5ef30 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -64,6 +64,15 @@ module API false end + def project_params + { + description: "", + path: Project.parse_project_id(project_match[:project_id]), + namespace_id: project_namespace&.id, + visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s + } + end + private def project_path_regex @@ -71,11 +80,13 @@ module API end def project_match - @match ||= params[:project].match(project_path_regex).captures + @project_match ||= params[:project].match(project_path_regex) end - def namespace - @namespace ||= Namespace.find_by_path_or_name(project_match[:namespace_id]) + def project_namespace + return unless project_match + + @project_namespace ||= Namespace.find_by_path_or_name(project_match[:namespace_id]) end # rubocop:disable Gitlab/ModuleWithInstanceVariables diff --git a/lib/api/internal.rb b/lib/api/internal.rb index f641ef457a3..b7475af2044 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -43,7 +43,7 @@ module API access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess access_checker = access_checker_klass - .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path, target_namespace: namespace) + .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path, target_namespace: project_namespace) begin access_checker.check(params[:action], params[:changes]) @@ -52,13 +52,6 @@ module API end if user && project.blank? && receive_pack? - project_params = { - description: "", - path: Project.parse_project_id(project_match[:project_name]), - namespace_id: namespace&.id, - visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s - } - @project = ::Projects::CreateService.new(user, project_params).execute if @project.saved? diff --git a/lib/gitlab/checks/new_project.rb b/lib/gitlab/checks/new_project.rb index 40d0acefaba..488c5c03c32 100644 --- a/lib/gitlab/checks/new_project.rb +++ b/lib/gitlab/checks/new_project.rb @@ -20,6 +20,8 @@ module Gitlab end def add_new_project_message + return unless user.present? && project.present? + Gitlab::Redis::SharedState.with do |redis| key = self.class.new_project_message_key(user.id, project.id) redis.setex(key, 5.minutes, new_project_message) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 598506aa418..38649a4fdef 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -19,8 +19,7 @@ module Gitlab upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', read_only: 'The repository is temporarily read-only. Please try again later.', - cannot_push_to_read_only: "You can't push code to a read-only GitLab instance.", - create: "Creating a repository to that namespace is not allowed." + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -53,7 +52,7 @@ module Gitlab check_download_access! when *PUSH_COMMANDS check_push_access!(cmd, changes) - check_repository_creation!(cmd) + check_namespace_accessibility!(cmd) end true @@ -149,16 +148,12 @@ module Gitlab end end - def check_repository_creation!(cmd) + def check_namespace_accessibility!(cmd) return unless project.blank? unless target_namespace raise NotFoundError, ERROR_MESSAGES[:namespace_not_found] end - - unless can_create_project_in_namespace?(cmd) - raise UnauthorizedError, ERROR_MESSAGES[:create] - end end def check_download_access! -- cgit v1.2.1 From e42a548f1dac02577d0c1731fef508dab68c45a5 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 25 Jan 2018 12:26:52 +0000 Subject: Move new project on push logic to a service --- lib/api/helpers/internal_helpers.rb | 29 ++++------------- lib/api/internal.rb | 16 ++++------ lib/gitlab/checks/new_project.rb | 62 ------------------------------------ lib/gitlab/checks/project_created.rb | 62 ++++++++++++++++++++++++++++++++++++ lib/gitlab/git_access.rb | 15 ++++++--- lib/gitlab/git_access_wiki.rb | 4 --- lib/gitlab/path_regex.rb | 4 +++ 7 files changed, 90 insertions(+), 102 deletions(-) delete mode 100644 lib/gitlab/checks/new_project.rb create mode 100644 lib/gitlab/checks/project_created.rb (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index fd568c5ef30..2340e962918 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -29,10 +29,6 @@ module API {} end - def receive_pack? - params[:action] == 'git-receive-pack' - end - def fix_git_env_repository_paths(env, repository_path) if obj_dir_relative = env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence env['GIT_OBJECT_DIRECTORY'] = File.join(repository_path, obj_dir_relative) @@ -51,6 +47,10 @@ module API ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) end + def receive_pack? + params[:action] == 'git-receive-pack' + end + def merge_request_urls ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end @@ -64,29 +64,14 @@ module API false end - def project_params - { - description: "", - path: Project.parse_project_id(project_match[:project_id]), - namespace_id: project_namespace&.id, - visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s - } + def project_namespace + @project_namespace ||= project&.namespace || Namespace.find_by_full_path(project_match[:namespace_path]) end private - def project_path_regex - @project_regex ||= /\A(?#{Gitlab::PathRegex.full_namespace_route_regex})\/(?#{Gitlab::PathRegex.project_git_route_regex})\z/.freeze - end - def project_match - @project_match ||= params[:project].match(project_path_regex) - end - - def project_namespace - return unless project_match - - @project_namespace ||= Namespace.find_by_path_or_name(project_match[:namespace_id]) + @project_match ||= params[:project].match(Gitlab::PathRegex.full_project_git_path_regex) end # rubocop:disable Gitlab/ModuleWithInstanceVariables diff --git a/lib/api/internal.rb b/lib/api/internal.rb index b7475af2044..841a34eb67f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -51,13 +51,11 @@ module API return { status: false, message: e.message } end - if user && project.blank? && receive_pack? - @project = ::Projects::CreateService.new(user, project_params).execute - - if @project.saved? - Gitlab::Checks::NewProject.new(user, @project, protocol).add_new_project_message - else - return { status: false, message: "Could not create project" } + if receive_pack? && project.blank? + begin + @project = ::Projects::CreateFromPushService.new(user, project_match[:project_path], project_namespace, protocol).execute + rescue Gitlab::GitAccess::ProjectCreationError => e + return { status: false, message: e.message } end end @@ -218,10 +216,10 @@ module API # key could be used if user redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) - new_project_message = Gitlab::Checks::NewProject.fetch_new_project_message(user.id, project.id) + project_created_message = Gitlab::Checks::ProjectCreated.fetch_project_created_message(user.id, project.id) output[:redirected_message] = redirect_message if redirect_message - output[:new_project_message] = new_project_message if new_project_message + output[:project_created_message] = project_created_message if project_created_message end output diff --git a/lib/gitlab/checks/new_project.rb b/lib/gitlab/checks/new_project.rb deleted file mode 100644 index 488c5c03c32..00000000000 --- a/lib/gitlab/checks/new_project.rb +++ /dev/null @@ -1,62 +0,0 @@ -module Gitlab - module Checks - class NewProject - NEW_PROJECT = "new_project".freeze - - def initialize(user, project, protocol) - @user = user - @project = project - @protocol = protocol - end - - def self.fetch_new_project_message(user_id, project_id) - new_project_key = new_project_message_key(user_id, project_id) - - Gitlab::Redis::SharedState.with do |redis| - message = redis.get(new_project_key) - redis.del(new_project_key) - message - end - end - - def add_new_project_message - return unless user.present? && project.present? - - Gitlab::Redis::SharedState.with do |redis| - key = self.class.new_project_message_key(user.id, project.id) - redis.setex(key, 5.minutes, new_project_message) - end - end - - def new_project_message - <<~MESSAGE.strip_heredoc - - The private project #{project.full_path} was created. - - To configure the remote, run: - git remote add origin #{git_url} - - To view the project, visit: - #{project_url} - - MESSAGE - end - - private - - attr_reader :project, :user, :protocol - - def self.new_project_message_key(user_id, project_id) - "#{NEW_PROJECT}:#{user_id}:#{project_id}" - end - - def project_url - Gitlab::Routing.url_helpers.project_url(project) - end - - def git_url - protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - end - end - end -end diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb new file mode 100644 index 00000000000..f05e8b4a7e8 --- /dev/null +++ b/lib/gitlab/checks/project_created.rb @@ -0,0 +1,62 @@ +module Gitlab + module Checks + class ProjectCreated + PROJECT_CREATED = "project_created".freeze + + def initialize(user, project, protocol) + @user = user + @project = project + @protocol = protocol + end + + def self.fetch_project_created_message(user_id, project_id) + project_created_key = project_created_message_key(user_id, project_id) + + Gitlab::Redis::SharedState.with do |redis| + message = redis.get(project_created_key) + redis.del(project_created_key) + message + end + end + + def add_project_created_message + return unless user.present? && project.present? + + Gitlab::Redis::SharedState.with do |redis| + key = self.class.project_created_message_key(user.id, project.id) + redis.setex(key, 5.minutes, project_created_message) + end + end + + def project_created_message + <<~MESSAGE.strip_heredoc + + The private project #{project.full_path} was created. + + To configure the remote, run: + git remote add origin #{git_url} + + To view the project, visit: + #{project_url} + + MESSAGE + end + + private + + attr_reader :project, :user, :protocol + + def self.project_created_message_key(user_id, project_id) + "#{PROJECT_CREATED}:#{user_id}:#{project_id}" + end + + def project_url + Gitlab::Routing.url_helpers.project_url(project) + end + + def git_url + protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 38649a4fdef..32a2395a26b 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -4,6 +4,7 @@ module Gitlab class GitAccess UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) + ProjectCreationError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) ERROR_MESSAGES = { @@ -13,13 +14,13 @@ module Gitlab 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', - namespace_not_found: 'The namespace you were looking for could not be found.', account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', read_only: 'The repository is temporarily read-only. Please try again later.', - cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance.", + namespace_not_found: 'The namespace you were looking for could not be found.' }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -52,7 +53,7 @@ module Gitlab check_download_access! when *PUSH_COMMANDS check_push_access!(cmd, changes) - check_namespace_accessibility!(cmd) + check_namespace_accessibility! end true @@ -148,7 +149,7 @@ module Gitlab end end - def check_namespace_accessibility!(cmd) + def check_namespace_accessibility! return unless project.blank? unless target_namespace @@ -248,7 +249,7 @@ module Gitlab end def can_create_project_in_namespace?(cmd) - return false unless PUSH_COMMANDS.include?(cmd) && target_namespace + return false unless push?(cmd) && target_namespace user.can?(:create_projects, target_namespace) end @@ -265,6 +266,10 @@ module Gitlab command == 'git-receive-pack' end + def push?(cmd) + PUSH_COMMANDS.include?(cmd) + end + def upload_pack_disabled_over_http? !Gitlab.config.gitlab_shell.upload_pack end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index f679b5e8ed6..1c9477e84b2 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -25,10 +25,6 @@ module Gitlab true end - def check_repository_creation!(cmd) - # Method not used in wiki - end - def push_to_read_only_message ERROR_MESSAGES[:read_only] end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 7e5dfd33502..4a2db11a978 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -187,6 +187,10 @@ module Gitlab @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z} end + def full_project_git_path_regex + @full_project_git_path_regex ||= /\A(\/|)(?#{full_namespace_route_regex})\/(?#{project_git_route_regex})\z/.freeze + end + def full_namespace_format_regex @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze end -- cgit v1.2.1 From dc229c076cdc0ef6e7f3f74f6e462c22880ff08c Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Fri, 26 Jan 2018 14:28:08 +0000 Subject: Abstracts ProjectMoved and ProjectCreated into a BaseProject --- lib/api/helpers/internal_helpers.rb | 8 +++++-- lib/api/internal.rb | 4 ++-- lib/gitlab/checks/base_project.rb | 46 ++++++++++++++++++++++++++++++++++++ lib/gitlab/checks/project_created.rb | 39 ++++-------------------------- lib/gitlab/checks/project_moved.rb | 38 +++++++---------------------- lib/gitlab/git_access.rb | 34 ++++++++++---------------- lib/gitlab/path_regex.rb | 2 +- 7 files changed, 79 insertions(+), 92 deletions(-) create mode 100644 lib/gitlab/checks/base_project.rb (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 2340e962918..bff245fe9a2 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,6 +1,8 @@ module API module Helpers module InternalHelpers + include Gitlab::Utils::StrongMemoize + attr_reader :redirected_path def wiki? @@ -65,13 +67,15 @@ module API end def project_namespace - @project_namespace ||= project&.namespace || Namespace.find_by_full_path(project_match[:namespace_path]) + strong_memoize(:project_namespace) do + project&.namespace || Namespace.find_by_full_path(project_match[:namespace_path]) + end end private def project_match - @project_match ||= params[:project].match(Gitlab::PathRegex.full_project_git_path_regex) + @project_match ||= params[:project].match(Gitlab::PathRegex.full_project_git_path_regex) || {} end # rubocop:disable Gitlab/ModuleWithInstanceVariables diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 841a34eb67f..ed6d022df97 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -215,8 +215,8 @@ module API # A user is not guaranteed to be returned; an orphaned write deploy # key could be used if user - redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) - project_created_message = Gitlab::Checks::ProjectCreated.fetch_project_created_message(user.id, project.id) + redirect_message = Gitlab::Checks::ProjectMoved.fetch_message(user.id, project.id) + project_created_message = Gitlab::Checks::ProjectCreated.fetch_message(user.id, project.id) output[:redirected_message] = redirect_message if redirect_message output[:project_created_message] = project_created_message if project_created_message diff --git a/lib/gitlab/checks/base_project.rb b/lib/gitlab/checks/base_project.rb new file mode 100644 index 00000000000..dd6c007b356 --- /dev/null +++ b/lib/gitlab/checks/base_project.rb @@ -0,0 +1,46 @@ +module Gitlab + module Checks + class BaseProject + def initialize(project, user, protocol) + @project = project + @user = user + @protocol = protocol + end + + def self.fetch_message(user_id, project_id) + key = message_key(user_id, project_id) + + Gitlab::Redis::SharedState.with do |redis| + message = redis.get(key) + redis.del(key) + message + end + end + + def add_message + return unless user.present? && project.present? + + Gitlab::Redis::SharedState.with do |redis| + key = self.class.message_key(user.id, project.id) + redis.setex(key, 5.minutes, message) + end + end + + def message + raise NotImplementedError + end + + protected + + attr_reader :project, :user, :protocol + + def self.message_key(user_id, project_id) + raise NotImplementedError + end + + def url_to_repo + protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + end + end + end +end diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb index f05e8b4a7e8..bd1e204bc81 100644 --- a/lib/gitlab/checks/project_created.rb +++ b/lib/gitlab/checks/project_created.rb @@ -1,40 +1,15 @@ module Gitlab module Checks - class ProjectCreated + class ProjectCreated < BaseProject PROJECT_CREATED = "project_created".freeze - def initialize(user, project, protocol) - @user = user - @project = project - @protocol = protocol - end - - def self.fetch_project_created_message(user_id, project_id) - project_created_key = project_created_message_key(user_id, project_id) - - Gitlab::Redis::SharedState.with do |redis| - message = redis.get(project_created_key) - redis.del(project_created_key) - message - end - end - - def add_project_created_message - return unless user.present? && project.present? - - Gitlab::Redis::SharedState.with do |redis| - key = self.class.project_created_message_key(user.id, project.id) - redis.setex(key, 5.minutes, project_created_message) - end - end - - def project_created_message + def message <<~MESSAGE.strip_heredoc The private project #{project.full_path} was created. To configure the remote, run: - git remote add origin #{git_url} + git remote add origin #{url_to_repo} To view the project, visit: #{project_url} @@ -44,19 +19,13 @@ module Gitlab private - attr_reader :project, :user, :protocol - - def self.project_created_message_key(user_id, project_id) + def self.message_key(user_id, project_id) "#{PROJECT_CREATED}:#{user_id}:#{project_id}" end def project_url Gitlab::Routing.url_helpers.project_url(project) end - - def git_url - protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - end end end end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index dfb2f4d4054..eca59e88e24 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -1,37 +1,15 @@ module Gitlab module Checks - class ProjectMoved + class ProjectMoved < BaseProject REDIRECT_NAMESPACE = "redirect_namespace".freeze - def initialize(project, user, redirected_path, protocol) - @project = project - @user = user + def initialize(project, user, protocol, redirected_path) @redirected_path = redirected_path - @protocol = protocol - end - - def self.fetch_redirect_message(user_id, project_id) - redirect_key = redirect_message_key(user_id, project_id) - Gitlab::Redis::SharedState.with do |redis| - message = redis.get(redirect_key) - redis.del(redirect_key) - message - end - end - - def add_redirect_message - # Don't bother with sending a redirect message for anonymous clones - # because they never see it via the `/internal/post_receive` endpoint - return unless user.present? && project.present? - - Gitlab::Redis::SharedState.with do |redis| - key = self.class.redirect_message_key(user.id, project.id) - redis.setex(key, 5.minutes, redirect_message) - end + super(project, user, protocol) end - def redirect_message(rejected: false) + def message(rejected: false) <<~MESSAGE.strip_heredoc Project '#{redirected_path}' was moved to '#{project.full_path}'. @@ -47,17 +25,17 @@ module Gitlab private - attr_reader :project, :redirected_path, :protocol, :user + attr_reader :redirected_path - def self.redirect_message_key(user_id, project_id) + def self.message_key(user_id, project_id) "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" end def remote_url_message(rejected) if rejected - "git remote set-url origin #{url} and try again." + "git remote set-url origin #{url_to_repo} and try again." else - "git remote set-url origin #{url}" + "git remote set-url origin #{url_to_repo}" end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 32a2395a26b..3d07e112e2b 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -19,8 +19,7 @@ module Gitlab upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', read_only: 'The repository is temporarily read-only. Please try again later.', - cannot_push_to_read_only: "You can't push code to a read-only GitLab instance.", - namespace_not_found: 'The namespace you were looking for could not be found.' + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -53,7 +52,6 @@ module Gitlab check_download_access! when *PUSH_COMMANDS check_push_access!(cmd, changes) - check_namespace_accessibility! end true @@ -100,7 +98,7 @@ module Gitlab end def check_project_accessibility!(cmd) - if (project.blank? || !can_read_project?) && !can_create_project_in_namespace?(cmd) + unless can_create_project_in_namespace?(cmd) || can_read_project? raise NotFoundError, ERROR_MESSAGES[:project_not_found] end end @@ -108,12 +106,12 @@ module Gitlab def check_project_moved! return if redirected_path.nil? - project_moved = Checks::ProjectMoved.new(project, user, redirected_path, protocol) + project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path) if project_moved.permanent_redirect? - project_moved.add_redirect_message + project_moved.add_message else - raise ProjectMovedError, project_moved.redirect_message(rejected: true) + raise ProjectMovedError, project_moved.message(rejected: true) end end @@ -144,19 +142,11 @@ module Gitlab end def check_repository_existence!(cmd) - if (project.blank? || !project.repository.exists?) && !can_create_project_in_namespace?(cmd) + unless can_create_project_in_namespace?(cmd) || project.repository.exists? raise UnauthorizedError, ERROR_MESSAGES[:no_repo] end end - def check_namespace_accessibility! - return unless project.blank? - - unless target_namespace - raise NotFoundError, ERROR_MESSAGES[:namespace_not_found] - end - end - def check_download_access! return if deploy_key? @@ -170,16 +160,16 @@ module Gitlab end def check_push_access!(cmd, changes) - return if project.blank? && can_create_project_in_namespace?(cmd) + if Gitlab::Database.read_only? + raise UnauthorizedError, push_to_read_only_message + end + + return if can_create_project_in_namespace?(cmd) if project.repository_read_only? raise UnauthorizedError, ERROR_MESSAGES[:read_only] end - if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message - end - if deploy_key check_deploy_key_push_access! elsif user @@ -249,7 +239,7 @@ module Gitlab end def can_create_project_in_namespace?(cmd) - return false unless push?(cmd) && target_namespace + return false unless push?(cmd) && target_namespace && project.blank? user.can?(:create_projects, target_namespace) end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 4a2db11a978..c6a594d38d1 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -188,7 +188,7 @@ module Gitlab end def full_project_git_path_regex - @full_project_git_path_regex ||= /\A(\/|)(?#{full_namespace_route_regex})\/(?#{project_git_route_regex})\z/.freeze + @full_project_git_path_regex ||= /\A\/?(?#{full_namespace_route_regex})\/(?#{project_git_route_regex})\z/.freeze end def full_namespace_format_regex -- cgit v1.2.1 From 839829a7786dd163eccb470bf251211bfb90bd72 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Wed, 31 Jan 2018 13:52:46 +0000 Subject: Adds documentation for the feature --- lib/gitlab/git_access.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 3d07e112e2b..84299dd5790 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,6 +2,8 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess + include Gitlab::Utils::StrongMemoize + UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) @@ -239,9 +241,11 @@ module Gitlab end def can_create_project_in_namespace?(cmd) - return false unless push?(cmd) && target_namespace && project.blank? + strong_memoize(:can_create_project_in_namespace) do + return false unless push?(cmd) && target_namespace && project.blank? - user.can?(:create_projects, target_namespace) + user.can?(:create_projects, target_namespace) + end end def http? -- cgit v1.2.1 From 28ef8cc56ec2f3b48e383050671b9dece58fe039 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Tue, 6 Feb 2018 15:37:52 +0200 Subject: Add sorting options for /users API (admin only) Signed-off-by: Dmitriy Zaporozhets --- lib/api/users.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/api/users.rb b/lib/api/users.rb index c7c2aa280d5..3cc12724b8a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -18,6 +18,14 @@ module API User.find_by(id: id) || not_found!('User') end + def reorder_users(users) + if params[:order_by] && params[:sort] + users.reorder(params[:order_by] => params[:sort]) + else + users + end + end + params :optional_attributes do optional :skype, type: String, desc: 'The Skype username' optional :linkedin, type: String, desc: 'The LinkedIn username' @@ -35,6 +43,13 @@ module API optional :avatar, type: File, desc: 'Avatar image for user' all_or_none_of :extern_uid, :provider end + + params :sort_params do + optional :order_by, type: String, values: %w[id name username created_at updated_at], + default: 'id', desc: 'Return users ordered by a field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return users sorted in ascending and descending order' + end end desc 'Get the list of users' do @@ -53,16 +68,18 @@ module API optional :created_before, type: DateTime, desc: 'Return users created before the specified time' all_or_none_of :extern_uid, :provider + use :sort_params use :pagination end get do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) unless current_user&.admin? - params.except!(:created_after, :created_before) + params.except!(:created_after, :created_before, :order_by, :sort) end users = UsersFinder.new(current_user, params).execute + users = reorder_users(users) authorized = can?(current_user, :read_users_list) -- cgit v1.2.1 From 11f882ca437337ebd04331b9e96217f1d48be93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 6 Feb 2018 15:31:39 +0100 Subject: Upgrade GitLab Workhorse to v3.6.0 --- lib/gitlab/workhorse.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index b3f8b0d174d..823df67ea39 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -161,6 +161,18 @@ module Gitlab ] end + def send_url(url, allow_redirects: false) + params = { + 'URL' => url, + 'AllowRedirects' => allow_redirects + } + + [ + SEND_DATA_HEADER, + "send-url:#{encode(params)}" + ] + end + def terminal_websocket(terminal) details = { 'Terminal' => { -- cgit v1.2.1 From 7381944565701f2a8db5d58d5bc3c7e52e7f60bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarka=20Kadlecova=CC=81?= Date: Wed, 31 Jan 2018 15:59:59 +0100 Subject: Support search in API --- lib/api/api.rb | 1 + lib/api/entities.rb | 25 ++++---- lib/api/search.rb | 110 +++++++++++++++++++++++++++++++++++ lib/api/v3/projects.rb | 2 +- lib/gitlab/project_search_results.rb | 7 ++- lib/gitlab/search_results.rb | 10 ++-- lib/gitlab/snippet_search_results.rb | 4 +- 7 files changed, 137 insertions(+), 22 deletions(-) create mode 100644 lib/api/search.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index f3f64244589..e953f3d2eca 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -146,6 +146,7 @@ module API mount ::API::Repositories mount ::API::Runner mount ::API::Runners + mount ::API::Search mount ::API::Services mount ::API::Settings mount ::API::SidekiqMetrics diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e13463ec66b..7838de13c56 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -314,24 +314,20 @@ module API end end - class ProjectSnippet < Grape::Entity + class Snippet < Grape::Entity expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at - - expose :web_url do |snippet, options| + expose :project_id + expose :web_url do |snippet| Gitlab::UrlBuilder.build(snippet) end end - class PersonalSnippet < Grape::Entity - expose :id, :title, :file_name, :description - expose :author, using: Entities::UserBasic - expose :updated_at, :created_at + class ProjectSnippet < Snippet + end - expose :web_url do |snippet| - Gitlab::UrlBuilder.build(snippet) - end + class PersonalSnippet < Snippet expose :raw_url do |snippet| Gitlab::UrlBuilder.build(snippet) + "/raw" end @@ -1168,5 +1164,14 @@ module API class ApplicationWithSecret < Application expose :secret end + + class Blob < Grape::Entity + expose :basename + expose :data + expose :filename + expose :id + expose :ref + expose :startline + end end end diff --git a/lib/api/search.rb b/lib/api/search.rb new file mode 100644 index 00000000000..31121b3ee2d --- /dev/null +++ b/lib/api/search.rb @@ -0,0 +1,110 @@ +module API + class Search < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + SCOPE_ENTITY = { + merge_requests: Entities::MergeRequestBasic, + issues: Entities::IssueBasic, + projects: Entities::BasicProjectDetails, + milestones: Entities::Milestone, + notes: Entities::Note, + commits: Entities::Commit, + blobs: Entities::Blob, + wiki_blobs: Entities::Blob, + snippet_titles: Entities::Snippet, + snippet_blobs: Entities::Snippet + }.freeze + + def search(additional_params = {}) + search_params = { + scope: params[:scope], + search: params[:search], + snippets: snippets?, + page: params[:page], + per_page: params[:per_page], + without_counts: false + }.merge(additional_params) + + results = SearchService.new(current_user, search_params).search_objects + + process_results(results) + end + + def process_results(results) + case params[:scope] + when 'wiki_blobs' + paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob) } + when 'blobs' + paginate(results).map { |blob| blob[1] } + else + paginate(results) + end + end + + def snippets? + %w(snippet_blobs snippet_titles).include?(params[:scope]).to_s + end + + def entity + SCOPE_ENTITY[params[:scope].to_sym] + end + end + + resource :search do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, type: String, desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs', + values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs) + use :pagination + end + get do + present search, with: entity + end + end + + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :id, type: String, desc: 'The ID of a group' + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, type: String, desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones', + values: %w(projects issues merge_requests milestones) + use :pagination + end + get ':id/-/search' do + find_group!(params[:id]) + + present search(group_id: params[:id]), with: entity + end + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, type: String, desc: 'The scope of search, available scopes: + issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs', + values: %w(issues merge_requests milestones notes wiki_blobs commits blobs) + use :pagination + end + get ':id/-/search' do + find_project!(params[:id]) + + present search(project_id: params[:id]), with: entity + end + end + end +end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index c856ba99f09..7d8b1f369fe 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -174,7 +174,7 @@ module API use :pagination end get "/search/:query", requirements: { query: %r{[^/]+} } do - search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + search_service = ::Search::GlobalService.new(current_user, search: params[:query]).execute projects = search_service.objects('projects', params[:page], false) projects = projects.reorder(params[:order_by] => params[:sort]) diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 4823f703ba4..0846fdc4de3 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -2,14 +2,15 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(current_user, project, query, repository_ref = nil) + def initialize(current_user, project, query, repository_ref = nil, per_page: 20) @current_user = current_user @project = project @repository_ref = repository_ref.presence || project.default_branch @query = query + @per_page = per_page end - def objects(scope, page = nil) + def objects(scope, page = nil, without_counts = true) case scope when 'notes' notes.page(page).per(per_page) @@ -20,7 +21,7 @@ module Gitlab when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) else - super(scope, page, false) + super(scope, page, without_counts) end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7362514167f..5ad219179f3 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -10,6 +10,7 @@ module Gitlab @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) @data = opts.fetch(:data, nil) + @per_page = opts.fetch(:per_page, 20) end def path @@ -21,7 +22,7 @@ module Gitlab end end - attr_reader :current_user, :query + attr_reader :current_user, :query, :per_page # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -33,11 +34,12 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, limit_projects, query, default_project_filter: false) + def initialize(current_user, limit_projects, query, default_project_filter: false, per_page: 20) @current_user = current_user @limit_projects = limit_projects || Project.all @query = query @default_project_filter = default_project_filter + @per_page = per_page end def objects(scope, page = nil, without_count = true) @@ -153,10 +155,6 @@ module Gitlab 'projects' end - def per_page - 20 - end - def project_ids_relation limit_projects.select(:id).reorder(nil) end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 4f86b3e8f73..98c43475303 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -9,14 +9,14 @@ module Gitlab @query = query end - def objects(scope, page = nil) + def objects(scope, page = nil, without_counts = true) case scope when 'snippet_titles' snippet_titles.page(page).per(per_page) when 'snippet_blobs' snippet_blobs.page(page).per(per_page) else - super(scope, nil, false) + super(scope, nil, without_counts) end end -- cgit v1.2.1 From d2a77094ae4a44b63fbe22ca910e836cb336a729 Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Thu, 11 Jan 2018 23:12:34 +0000 Subject: File upload UI obeys LFS filters Uses Lfs::FileModificationHandler to coordinate LFS detection, creation of LfsObject, etc Caveats: 1. This isn't used by the multi-file editor / Web IDE 2. This isn't used on rename. We'd need to be able to download LFS files and add them to the commit if they no longer match so not as simple. 3. We only check the root .gitattributes file, so this should be improved to correctly check for nested .gitattributes files in subfolders. --- lib/carrier_wave_string_file.rb | 5 +++++ lib/gitlab/git/lfs_pointer_file.rb | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 lib/carrier_wave_string_file.rb create mode 100644 lib/gitlab/git/lfs_pointer_file.rb (limited to 'lib') diff --git a/lib/carrier_wave_string_file.rb b/lib/carrier_wave_string_file.rb new file mode 100644 index 00000000000..6c848902e4a --- /dev/null +++ b/lib/carrier_wave_string_file.rb @@ -0,0 +1,5 @@ +class CarrierWaveStringFile < StringIO + def original_filename + "" + end +end diff --git a/lib/gitlab/git/lfs_pointer_file.rb b/lib/gitlab/git/lfs_pointer_file.rb new file mode 100644 index 00000000000..da12ed7d125 --- /dev/null +++ b/lib/gitlab/git/lfs_pointer_file.rb @@ -0,0 +1,25 @@ +module Gitlab + module Git + class LfsPointerFile + def initialize(data) + @data = data + end + + def pointer + @pointer ||= <<~FILE + version https://git-lfs.github.com/spec/v1 + oid sha256:#{sha256} + size #{size} + FILE + end + + def size + @size ||= @data.bytesize + end + + def sha256 + @sha256 ||= Digest::SHA256.hexdigest(@data) + end + end + end +end -- cgit v1.2.1 From 285d5d526b1d568580b63488b8832cfc9ef19077 Mon Sep 17 00:00:00 2001 From: Bastian Blank Date: Mon, 5 Feb 2018 17:36:51 +0000 Subject: Close low level rugged repository in project cache worker Signed-off-by: Bastian Blank --- lib/gitlab/git/repository.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ab1362a3bb0..09fb3999d83 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -128,6 +128,10 @@ module Gitlab raise NoRepository.new('no repository for such path') end + def cleanup + @rugged&.close + end + def circuit_breaker @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end -- cgit v1.2.1 From 1e56b3f476f9779ec747534e94156a6b8076209c Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Fri, 2 Feb 2018 15:27:30 +0000 Subject: Moves project creationg to git access check for git push --- lib/api/helpers/internal_helpers.rb | 20 +++----- lib/api/internal.rb | 15 ++---- lib/gitlab/checks/base_project.rb | 46 ----------------- lib/gitlab/checks/post_push_message.rb | 46 +++++++++++++++++ lib/gitlab/checks/project_created.rb | 6 +-- lib/gitlab/checks/project_moved.rb | 4 +- lib/gitlab/git_access.rb | 92 +++++++++++++++++++++------------- lib/gitlab/path_regex.rb | 2 +- lib/gitlab/user_access.rb | 3 +- 9 files changed, 123 insertions(+), 111 deletions(-) delete mode 100644 lib/gitlab/checks/base_project.rb create mode 100644 lib/gitlab/checks/post_push_message.rb (limited to 'lib') diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index bff245fe9a2..cd59da6fc70 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,8 +1,6 @@ module API module Helpers module InternalHelpers - include Gitlab::Utils::StrongMemoize - attr_reader :redirected_path def wiki? @@ -49,10 +47,6 @@ module API ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) end - def receive_pack? - params[:action] == 'git-receive-pack' - end - def merge_request_urls ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end @@ -66,16 +60,18 @@ module API false end - def project_namespace - strong_memoize(:project_namespace) do - project&.namespace || Namespace.find_by_full_path(project_match[:namespace_path]) - end + def project_path + project&.path || project_path_match[:project_path] + end + + def namespace_path + project&.namespace&.full_path || project_path_match[:namespace_path] end private - def project_match - @project_match ||= params[:project].match(Gitlab::PathRegex.full_project_git_path_regex) || {} + def project_path_match + @project_path_match ||= params[:project].match(Gitlab::PathRegex.full_project_git_path_regex) || {} end # rubocop:disable Gitlab/ModuleWithInstanceVariables diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ed6d022df97..9285fb90cdc 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -42,23 +42,18 @@ module API end access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess - access_checker = access_checker_klass - .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path, target_namespace: project_namespace) + access_checker = access_checker_klass.new(actor, project, + protocol, authentication_abilities: ssh_authentication_abilities, + namespace_path: namespace_path, project_path: project_path, + redirected_path: redirected_path) begin access_checker.check(params[:action], params[:changes]) + @project ||= access_checker.project rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e return { status: false, message: e.message } end - if receive_pack? && project.blank? - begin - @project = ::Projects::CreateFromPushService.new(user, project_match[:project_path], project_namespace, protocol).execute - rescue Gitlab::GitAccess::ProjectCreationError => e - return { status: false, message: e.message } - end - end - log_user_activity(actor) { diff --git a/lib/gitlab/checks/base_project.rb b/lib/gitlab/checks/base_project.rb deleted file mode 100644 index dd6c007b356..00000000000 --- a/lib/gitlab/checks/base_project.rb +++ /dev/null @@ -1,46 +0,0 @@ -module Gitlab - module Checks - class BaseProject - def initialize(project, user, protocol) - @project = project - @user = user - @protocol = protocol - end - - def self.fetch_message(user_id, project_id) - key = message_key(user_id, project_id) - - Gitlab::Redis::SharedState.with do |redis| - message = redis.get(key) - redis.del(key) - message - end - end - - def add_message - return unless user.present? && project.present? - - Gitlab::Redis::SharedState.with do |redis| - key = self.class.message_key(user.id, project.id) - redis.setex(key, 5.minutes, message) - end - end - - def message - raise NotImplementedError - end - - protected - - attr_reader :project, :user, :protocol - - def self.message_key(user_id, project_id) - raise NotImplementedError - end - - def url_to_repo - protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - end - end - end -end diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb new file mode 100644 index 00000000000..473c0385b34 --- /dev/null +++ b/lib/gitlab/checks/post_push_message.rb @@ -0,0 +1,46 @@ +module Gitlab + module Checks + class PostPushMessage + def initialize(project, user, protocol) + @project = project + @user = user + @protocol = protocol + end + + def self.fetch_message(user_id, project_id) + key = message_key(user_id, project_id) + + Gitlab::Redis::SharedState.with do |redis| + message = redis.get(key) + redis.del(key) + message + end + end + + def add_message + return unless user.present? && project.present? + + Gitlab::Redis::SharedState.with do |redis| + key = self.class.message_key(user.id, project.id) + redis.setex(key, 5.minutes, message) + end + end + + def message + raise NotImplementedError + end + + protected + + attr_reader :project, :user, :protocol + + def self.message_key(user_id, project_id) + raise NotImplementedError + end + + def url_to_repo + protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + end + end + end +end diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb index bd1e204bc81..cec270d6a58 100644 --- a/lib/gitlab/checks/project_created.rb +++ b/lib/gitlab/checks/project_created.rb @@ -1,12 +1,12 @@ module Gitlab module Checks - class ProjectCreated < BaseProject + class ProjectCreated < PostPushMessage PROJECT_CREATED = "project_created".freeze def message - <<~MESSAGE.strip_heredoc + <<~MESSAGE - The private project #{project.full_path} was created. + The private project #{project.full_path} was successfully created. To configure the remote, run: git remote add origin #{url_to_repo} diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index eca59e88e24..3263790a876 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -1,6 +1,6 @@ module Gitlab module Checks - class ProjectMoved < BaseProject + class ProjectMoved < PostPushMessage REDIRECT_NAMESPACE = "redirect_namespace".freeze def initialize(project, user, protocol, redirected_path) @@ -10,7 +10,7 @@ module Gitlab end def message(rejected: false) - <<~MESSAGE.strip_heredoc + <<~MESSAGE Project '#{redirected_path}' was moved to '#{project.full_path}'. Please update your Git remote: diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 84299dd5790..bc1e83f77b2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -28,32 +28,37 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :redirected_path, :target_namespace + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path - def initialize(actor, project, protocol, authentication_abilities:, redirected_path: nil, target_namespace: nil) + def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil) @actor = actor @project = project @protocol = protocol - @redirected_path = redirected_path @authentication_abilities = authentication_abilities - @target_namespace = target_namespace + @namespace_path = namespace_path + @project_path = project_path + @redirected_path = redirected_path end def check(cmd, changes) check_protocol! check_valid_actor! check_active_user! - check_project_accessibility!(cmd) - check_project_moved! check_command_disabled!(cmd) check_command_existence!(cmd) - check_repository_existence!(cmd) + check_db_accessibility!(cmd) + + ensure_project_on_push!(cmd, changes) + + check_project_accessibility! + check_project_moved! + check_repository_existence! case cmd when *DOWNLOAD_COMMANDS check_download_access! when *PUSH_COMMANDS - check_push_access!(cmd, changes) + check_push_access!(changes) end true @@ -99,8 +104,8 @@ module Gitlab end end - def check_project_accessibility!(cmd) - unless can_create_project_in_namespace?(cmd) || can_read_project? + def check_project_accessibility! + if project.blank? || !can_read_project? raise NotFoundError, ERROR_MESSAGES[:project_not_found] end end @@ -143,16 +148,49 @@ module Gitlab end end - def check_repository_existence!(cmd) - unless can_create_project_in_namespace?(cmd) || project.repository.exists? + def check_db_accessibility!(cmd) + return unless receive_pack?(cmd) + + if Gitlab::Database.read_only? + raise UnauthorizedError, push_to_read_only_message + end + end + + def ensure_project_on_push!(cmd, changes) + return if project || deploy_key? + return unless receive_pack?(cmd) && changes == '_any' && authentication_abilities.include?(:push_code) + + namespace = Namespace.find_by_full_path(namespace_path) + + return unless user&.can?(:create_projects, namespace) + + project_params = { + path: project_path, + namespace_id: namespace.id, + visibility_level: Gitlab::VisibilityLevel::PRIVATE + } + + project = Projects::CreateService.new(user, project_params).execute + + unless project.saved? + raise ProjectCreationError, "Could not create project: #{project.errors.full_messages.join(', ')}" + end + + @project = project + user_access.project = @project + + Checks::ProjectCreated.new(project, user, protocol).add_message + end + + def check_repository_existence! + unless project.repository.exists? raise UnauthorizedError, ERROR_MESSAGES[:no_repo] end end def check_download_access! - return if deploy_key? - - passed = user_can_download_code? || + passed = deploy_key? || + user_can_download_code? || build_can_download_code? || guest_can_download_code? @@ -161,13 +199,7 @@ module Gitlab end end - def check_push_access!(cmd, changes) - if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message - end - - return if can_create_project_in_namespace?(cmd) - + def check_push_access!(changes) if project.repository_read_only? raise UnauthorizedError, ERROR_MESSAGES[:read_only] end @@ -180,8 +212,6 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:upload] end - return if changes.blank? # Allow access. - check_change_access!(changes) end @@ -198,6 +228,8 @@ module Gitlab end def check_change_access!(changes) + return if changes.blank? # Allow access. + changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied @@ -240,14 +272,6 @@ module Gitlab end || Guest.can?(:read_project, project) end - def can_create_project_in_namespace?(cmd) - strong_memoize(:can_create_project_in_namespace) do - return false unless push?(cmd) && target_namespace && project.blank? - - user.can?(:create_projects, target_namespace) - end - end - def http? protocol == 'http' end @@ -260,10 +284,6 @@ module Gitlab command == 'git-receive-pack' end - def push?(cmd) - PUSH_COMMANDS.include?(cmd) - end - def upload_pack_disabled_over_http? !Gitlab.config.gitlab_shell.upload_pack end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index c6a594d38d1..1fc0363904a 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -188,7 +188,7 @@ module Gitlab end def full_project_git_path_regex - @full_project_git_path_regex ||= /\A\/?(?#{full_namespace_route_regex})\/(?#{project_git_route_regex})\z/.freeze + @full_project_git_path_regex ||= %r{\A\/?(?#{full_namespace_route_regex})\/(?#{project_route_regex})\.git\z} end def full_namespace_format_regex diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index f357488ac61..15eb1c41213 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -6,7 +6,8 @@ module Gitlab [user&.id, project&.id] end - attr_reader :user, :project + attr_reader :user + attr_accessor :project def initialize(user, project: nil) @user = user -- cgit v1.2.1 From 1b2400b529628da08e406e2391ef37c0b05f889e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarka=20Kadlecov=C3=A1?= Date: Tue, 6 Feb 2018 17:53:42 +0100 Subject: Return only limited pagination headers for search API endpoints --- lib/api/helpers/pagination.rb | 17 +++++++++++++---- lib/api/search.rb | 3 +-- lib/gitlab/project_search_results.rb | 4 ++-- lib/gitlab/snippet_search_results.rb | 4 ++-- 4 files changed, 18 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index bb70370ba77..09805049169 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -12,13 +12,16 @@ module API private def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', total_pages(paginated_data).to_s header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s header 'X-Prev-Page', paginated_data.prev_page.to_s header 'Link', pagination_links(paginated_data) + + return if data_without_counts?(paginated_data) + + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', total_pages(paginated_data).to_s end def pagination_links(paginated_data) @@ -37,8 +40,10 @@ module API request_params[:page] = 1 links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - request_params[:page] = total_pages(paginated_data) - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + unless data_without_counts?(paginated_data) + request_params[:page] = total_pages(paginated_data) + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + end links.join(', ') end @@ -55,6 +60,10 @@ module API relation end + + def data_without_counts?(paginated_data) + paginated_data.is_a?(Kaminari::PaginatableWithoutCount) + end end end end diff --git a/lib/api/search.rb b/lib/api/search.rb index 31121b3ee2d..3912c66657e 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -24,8 +24,7 @@ module API search: params[:search], snippets: snippets?, page: params[:page], - per_page: params[:per_page], - without_counts: false + per_page: params[:per_page] }.merge(additional_params) results = SearchService.new(current_user, search_params).search_objects diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0846fdc4de3..9e2fa07a205 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -10,7 +10,7 @@ module Gitlab @per_page = per_page end - def objects(scope, page = nil, without_counts = true) + def objects(scope, page = nil) case scope when 'notes' notes.page(page).per(per_page) @@ -21,7 +21,7 @@ module Gitlab when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) else - super(scope, page, without_counts) + super(scope, page, false) end end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 98c43475303..4f86b3e8f73 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -9,14 +9,14 @@ module Gitlab @query = query end - def objects(scope, page = nil, without_counts = true) + def objects(scope, page = nil) case scope when 'snippet_titles' snippet_titles.page(page).per(per_page) when 'snippet_blobs' snippet_blobs.page(page).per(per_page) else - super(scope, nil, without_counts) + super(scope, nil, false) end end -- cgit v1.2.1 From 876854285f694d47430f348c0946be5f3e21e12a Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 6 Feb 2018 11:14:45 -0600 Subject: Explicitly set cwd in Sidekiq memory killer instead of depending on getcwd --- lib/gitlab/sidekiq_middleware/memory_killer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 2bfb7caefd9..b89ae2505c9 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -45,7 +45,7 @@ module Gitlab private def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid})) + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) return 0 unless status.zero? output.to_i -- cgit v1.2.1 From a03d29da1dbbef3c202899cef3cd89b453a8b76f Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 31 Jan 2018 11:39:03 -0600 Subject: Validate User username only on Namespace, and bubble up appropriately --- lib/constraints/user_url_constrainer.rb | 2 +- lib/gitlab/o_auth/user.rb | 2 +- lib/gitlab/path_regex.rb | 12 ------------ 3 files changed, 2 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index b7633aa7cbb..3b3ed1c6ddb 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -2,7 +2,7 @@ class UserUrlConstrainer def matches?(request) full_path = request.params[:username] - return false unless UserPathValidator.valid_path?(full_path) + return false unless NamespacePathValidator.valid_path?(full_path) User.find_by_full_path(full_path, follow_redirects: request.get?).present? end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e40a001d20c..a3e1c66c19f 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -178,7 +178,7 @@ module Gitlab valid_username = ::Namespace.clean_path(username) uniquify = Uniquify.new - valid_username = uniquify.string(valid_username) { |s| !UserPathValidator.valid_path?(s) } + valid_username = uniquify.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } name = auth_hash.name name = valid_username if name.strip.empty? diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 7e5dfd33502..ef3f786686a 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -171,26 +171,14 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end - def root_namespace_path_regex - @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z} - end - def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end - def project_path_regex - @project_path_regex ||= %r{\A#{project_route_regex}/\z} - end - def full_project_path_regex @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z} end - def full_namespace_format_regex - @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze - end - def namespace_format_regex @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/.freeze end -- cgit v1.2.1 From 81b5ce11525c93c479041a31257f522f03c9d129 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 6 Feb 2018 12:19:48 -0600 Subject: Only set cwd on the newly spawned process, instead of the current one --- lib/gitlab/git/hook.rb | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index e29a1f7afa1..24f027d8da4 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -82,14 +82,20 @@ module Gitlab end def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - Dir.chdir(repo_path) do - env = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username - } - stdout, stderr, status = Open3.capture3(env, path, ref, oldrev, newrev) - [status.success?, (stderr.presence || stdout).gsub(/\R/, "
").html_safe] - end + env = { + 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username, + 'PWD' => repo_path + } + + options = { + chdir: repo_path + } + + args = [ref, oldrev, newrev] + + stdout, stderr, status = Open3.capture3(env, path, *args, options) + [status.success?, (stderr.presence || stdout).gsub(/\R/, "
").html_safe] end def retrieve_error_message(stderr, stdout) -- cgit v1.2.1 From 8b4280cb25cd27c3b2c1cbdfb7ee871a7ebaa6d3 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Tue, 6 Feb 2018 17:25:36 +0000 Subject: Check ability ability before proceeding with project specific checks --- lib/gitlab/git_access.rb | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index bc1e83f77b2..8ec3386184a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -12,8 +12,9 @@ module Gitlab ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', - deploy_key_upload: - 'This deploy key does not have write access to this project.', + auth_upload: 'You are not allowed to upload code.', + auth_download: 'You are not allowed to download code.', + deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', account_blocked: 'Your account has been blocked.', @@ -44,6 +45,7 @@ module Gitlab check_protocol! check_valid_actor! check_active_user! + check_authentication_abilities!(cmd) check_command_disabled!(cmd) check_command_existence!(cmd) check_db_accessibility!(cmd) @@ -104,6 +106,19 @@ module Gitlab end end + def check_authentication_abilities!(cmd) + case cmd + when *DOWNLOAD_COMMANDS + unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code) + raise UnauthorizedError, ERROR_MESSAGES[:auth_download] + end + when *PUSH_COMMANDS + unless authentication_abilities.include?(:push_code) + raise UnauthorizedError, ERROR_MESSAGES[:auth_upload] + end + end + end + def check_project_accessibility! if project.blank? || !can_read_project? raise NotFoundError, ERROR_MESSAGES[:project_not_found] @@ -205,31 +220,21 @@ module Gitlab end if deploy_key - check_deploy_key_push_access! + unless deploy_key.can_push_to?(project) + raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] + end elsif user - check_user_push_access! + # User access is verified in check_change_access! else raise UnauthorizedError, ERROR_MESSAGES[:upload] end - check_change_access!(changes) - end + return if changes.blank? # Allow access this is needed for EE. - def check_user_push_access! - unless authentication_abilities.include?(:push_code) - raise UnauthorizedError, ERROR_MESSAGES[:upload] - end - end - - def check_deploy_key_push_access! - unless deploy_key.can_push_to?(project) - raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] - end + check_change_access!(changes) end def check_change_access!(changes) - return if changes.blank? # Allow access. - changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied -- cgit v1.2.1 From 66d41d2c22ca949cb1877818b923109c914dffa9 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Mon, 5 Feb 2018 13:56:16 +1100 Subject: Handle all YAML parser exceptions in .gitlab-ci.yml (fixes #41209) - Move the exception handling as close to the source as possible to avoid leaking Psych ahstraction - Also remove unnecessary rescue all statement from LintsController. This should not be necessary anymore since any YAML errors should all be caught by the #validation_message method. --- lib/gitlab/ci/config/loader.rb | 2 ++ lib/gitlab/ci/yaml_processor.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb index e7d9f6a7761..141d2714cb6 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/ci/config/loader.rb @@ -6,6 +6,8 @@ module Gitlab def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) + rescue Psych::Exception => e + raise FormatError, e.message end def valid? diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 0bd78b03448..a7285ac8f9d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -85,7 +85,7 @@ module Gitlab begin Gitlab::Ci::YamlProcessor.new(content) nil - rescue ValidationError, Psych::SyntaxError => e + rescue ValidationError => e e.message end end -- cgit v1.2.1 From 277f7fef2c7369dc9fc8f54f9ad35a2d3086ee2b Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Wed, 7 Feb 2018 01:35:57 +0100 Subject: Make prometheus service querying approach much nicer wrt to arity and default function params --- lib/gitlab/prometheus/queries/matched_metrics_query.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index d21f64a252b..4c3edccc71a 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -4,7 +4,7 @@ module Gitlab class MatchedMetricsQuery < BaseQuery MAX_QUERY_ITEMS = 40.freeze - def query(_ = nil) + def query groups_data.map do |group, data| { group: group.name, -- cgit v1.2.1 From 0e90284c11815f84c804e7e922e709b31ca6d029 Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Wed, 7 Feb 2018 02:25:54 +0100 Subject: Catch json parsing error as PrometheusError --- lib/gitlab/prometheus_client.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 8264501b1ae..10527972663 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -42,6 +42,8 @@ module Gitlab def json_api_get(type, args = {}) path = ['api', 'v1', type].join('/') get(path, args) + rescue JSON::ParserError + raise PrometheusError, 'Parsing response failed' rescue Errno::ECONNREFUSED raise PrometheusError, 'Connection refused' end -- cgit v1.2.1 From 86e98c832a4eeaac616daef4ef9ddebeb7191fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarka=20Kadlecov=C3=A1?= Date: Wed, 7 Feb 2018 14:20:18 +0100 Subject: Small code/doc changes --- lib/api/search.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/search.rb b/lib/api/search.rb index 3912c66657e..9f08fd96a3b 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -58,8 +58,10 @@ module API end params do requires :search, type: String, desc: 'The expression it should be searched for' - requires :scope, type: String, desc: 'The scope of search, available scopes: - projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs', + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs', values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs) use :pagination end @@ -75,8 +77,10 @@ module API params do requires :id, type: String, desc: 'The ID of a group' requires :search, type: String, desc: 'The expression it should be searched for' - requires :scope, type: String, desc: 'The scope of search, available scopes: - projects, issues, merge_requests, milestones', + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones', values: %w(projects issues merge_requests milestones) use :pagination end @@ -94,8 +98,10 @@ module API params do requires :id, type: String, desc: 'The ID of a project' requires :search, type: String, desc: 'The expression it should be searched for' - requires :scope, type: String, desc: 'The scope of search, available scopes: - issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs', + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs', values: %w(issues merge_requests milestones notes wiki_blobs commits blobs) use :pagination end -- cgit v1.2.1 From bed948321173b49564f39837e97212ee4dd96e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20D=C3=A1vila?= Date: Wed, 7 Feb 2018 08:00:53 -0500 Subject: Backport of LFS File Locking API --- lib/gitlab/checks/change_access.rb | 29 +++++++++++++- lib/gitlab/checks/commit_check.rb | 61 ++++++++++++++++++++++++++++++ lib/gitlab/import_export/import_export.yml | 2 + 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/checks/commit_check.rb (limited to 'lib') diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 945d70e7a24..d75e73dac10 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -31,13 +31,14 @@ module Gitlab @protocol = protocol end - def exec + def exec(skip_commits_check: false) return true if skip_authorization push_checks branch_checks tag_checks lfs_objects_exist_check + commits_check unless skip_commits_check true end @@ -117,6 +118,24 @@ module Gitlab end end + def commits_check + return if deletion? || newrev.nil? + + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + commit_check.validate(commit, validations_for_commit(commit)) + end + end + + commit_check.validate_file_paths + end + + # Method overwritten in EE to inject custom validations + def validations_for_commit(_) + [] + end + private def updated_from_web? @@ -150,6 +169,14 @@ module Gitlab raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] end end + + def commit_check + @commit_check ||= Gitlab::Checks::CommitCheck.new(project, user_access.user, newrev, oldrev) + end + + def commits + project.repository.new_commits(newrev) + end end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb new file mode 100644 index 00000000000..ae0cd142378 --- /dev/null +++ b/lib/gitlab/checks/commit_check.rb @@ -0,0 +1,61 @@ +module Gitlab + module Checks + class CommitCheck + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :user, :newrev, :oldrev + + def initialize(project, user, newrev, oldrev) + @project = project + @user = user + @newrev = user + @oldrev = user + @file_paths = [] + end + + def validate(commit, validations) + return if validations.empty? && path_validations.empty? + + commit.raw_deltas.each do |diff| + @file_paths << (diff.new_path || diff.old_path) + + validations.each do |validation| + if error = validation.call(diff) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + end + + def validate_file_paths + path_validations.each do |validation| + if error = validation.call(@file_paths) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + + private + + def validate_lfs_file_locks? + strong_memoize(:validate_lfs_file_locks) do + project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev + end + end + + def lfs_file_locks_validation + lambda do |paths| + lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first + + if lfs_lock + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + end + end + end + + def path_validations + validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2daed10f678..9f404003125 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,6 +27,8 @@ project_tree: - :releases - project_members: - :user + - lfs_file_locks: + - :user - merge_requests: - notes: - :author -- cgit v1.2.1 From 73e78c4e1517dd50bd5dd0934e0f62288de2971b Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 31 Jan 2018 11:03:13 +0100 Subject: Don't use rugged in Repository#refs_hash The refs hash is used to determine what branches and tags have a commit as head in the network graph. The previous implementation depended on Rugged#references. The problem with this implementation was that it depended on rugged, but also that it iterated over all references and thus loading more data than needed if for example the project uses CI/CD environments, Pipelines, or Merge Requests. Given only refs are checked the network cares about the GraphHelper#refs method has no need to reject those, simplifying the method. Closes gitlab-org/gitaly#880 --- lib/gitlab/git/commit.rb | 20 ++++++++++---------- lib/gitlab/git/repository.rb | 25 +++++++++++-------------- 2 files changed, 21 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 768617e2cae..d95561fe1b2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -402,15 +402,6 @@ module Gitlab end end - # Get a collection of Rugged::Reference objects for this commit. - # - # Ex. - # commit.ref(repo) - # - def refs(repo) - repo.refs_hash[id] - end - # Get ref names collection # # Ex. @@ -418,7 +409,7 @@ module Gitlab # def ref_names(repo) refs(repo).map do |ref| - ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "") + ref.sub(%r{^refs/(heads|remotes|tags)/}, "") end end @@ -553,6 +544,15 @@ module Gitlab date: Google::Protobuf::Timestamp.new(seconds: author_or_committer[:time].to_i) ) end + + # Get a collection of Gitlab::Git::Ref objects for this commit. + # + # Ex. + # commit.ref(repo) + # + def refs(repo) + repo.refs_hash[id] + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ab1362a3bb0..39279960154 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -627,21 +627,18 @@ module Gitlab end end - # Get refs hash which key is SHA1 - # and value is a Rugged::Reference + # Get refs hash which key is is the commit id + # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch + # Note that both inherit from Gitlab::Git::Ref def refs_hash - # Initialize only when first call - if @refs_hash.nil? - @refs_hash = Hash.new { |h, k| h[k] = [] } - - rugged.references.each do |r| - # Symbolic/remote references may not have an OID; skip over them - target_oid = r.target.try(:oid) - if target_oid - sha = rev_parse_target(target_oid).oid - @refs_hash[sha] << r - end - end + return @refs_hash if @refs_hash + + @refs_hash = Hash.new { |h, k| h[k] = [] } + + (tags + branches).each do |ref| + next unless ref.target && ref.name + + @refs_hash[ref.dereferenced_target.id] << ref.name end @refs_hash -- cgit v1.2.1 From d40912bb4944d0ac9adae45f7c0621f40b996406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Wed, 7 Feb 2018 16:33:12 +0000 Subject: Removing gitaly flags --- lib/gitlab/git/wiki.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 39040d56971..ac12271a87e 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -25,9 +25,8 @@ module Gitlab @repository.exists? end - # Disabled because of https://gitlab.com/gitlab-org/gitaly/merge_requests/539 def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| if is_enabled gitaly_write_page(name, format, content, commit_details) gollum_wiki.clear_cache @@ -48,9 +47,8 @@ module Gitlab end end - # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42094 def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| if is_enabled gitaly_update_page(page_path, title, format, content, commit_details) gollum_wiki.clear_cache -- cgit v1.2.1