From 6d0c10b1b791e1938ce42c332280f5fe7dcd489f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 11 May 2018 13:28:51 +0200 Subject: Make it possible to compare untrusted regexps --- lib/gitlab/untrusted_regexp.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 7ce2e9d636e..fb25755391d 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -9,7 +9,7 @@ module Gitlab # there is a strict limit on total execution time. See the RE2 documentation # at https://github.com/google/re2/wiki/Syntax for more details. class UntrustedRegexp - delegate :===, to: :regexp + delegate :===, :source, to: :regexp def initialize(pattern) @regexp = RE2::Regexp.new(pattern, log_errors: false) @@ -31,6 +31,10 @@ module Gitlab RE2.Replace(text, regexp, rewrite) end + def ==(other) + self.source == other.source + end + private attr_reader :regexp -- cgit v1.2.1 From 8b736c91fc928157df9ace050f769d0948b58c1d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 11 May 2018 13:29:05 +0200 Subject: Implement variables expression untrusted pattern lexeme --- .../ci/pipeline/expression/lexeme/pattern.rb | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb new file mode 100644 index 00000000000..8cb1af30252 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -0,0 +1,26 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Pattern < Lexeme::Value + PATTERN = %r{/(?.+)/}.freeze + + def initialize(regexp) + @value = regexp + end + + def evaluate(variables = {}) + Gitlab::UntrustedRegexp.new(@value.to_s) + # TODO raise LexerError / ParserError in case of RegexpError + end + + def self.build(string) + new(string.match(PATTERN)[:regexp]) + end + end + end + end + end + end +end -- cgit v1.2.1 From be73838bdfa01b7a80d087e421505dccf625ae55 Mon Sep 17 00:00:00 2001 From: Chantal Rollison Date: Fri, 11 May 2018 11:05:14 -0700 Subject: Add anchor for incoming email regex, closes 44989 --- lib/gitlab/incoming_email.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index c9122a23568..d323cb9dadf 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -57,7 +57,7 @@ module Gitlab regex = Regexp.escape(wildcard_address) regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') - Regexp.new(regex).freeze + Regexp.new(/\A#{regex}\z/).freeze end end end -- cgit v1.2.1 From b784a985f2188e328da32cc9fdc73c8d4ac63733 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 14 May 2018 14:24:59 +0200 Subject: Do not raise if variable expression can not be evaluated --- lib/gitlab/ci/pipeline/expression/statement.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 09a7c98464b..363e0b708a6 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -35,6 +35,8 @@ module Gitlab def truthful? evaluate.present? + rescue StatementError + false end def valid? -- cgit v1.2.1 From ac65257c40052f739492f0648f6b7c06a1c95250 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 14 May 2018 14:38:08 +0200 Subject: Raise variables statement exception if pattern is invalid --- lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb | 6 ++++-- lib/gitlab/ci/pipeline/expression/parser.rb | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 8cb1af30252..2ff527e34a8 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -11,8 +11,10 @@ module Gitlab end def evaluate(variables = {}) - Gitlab::UntrustedRegexp.new(@value.to_s) - # TODO raise LexerError / ParserError in case of RegexpError + # TODO multiline support + @regexp = Gitlab::UntrustedRegexp.new(@value) + rescue RegexpError + raise Parser::ParserError, 'Invalid regular expression!' end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 90f94d0b763..fe23ab0b2f8 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -3,6 +3,8 @@ module Gitlab module Pipeline module Expression class Parser + ParserError = Class.new(Statement::StatementError) + def initialize(tokens) @tokens = tokens.to_enum @nodes = [] -- cgit v1.2.1 From f16f2b599412ed1514ba96d81758b9a2e6fd9c1f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 14 May 2018 15:03:10 +0200 Subject: Add pattern matching variables expression lexeme --- .../ci/pipeline/expression/lexeme/matches.rb | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 lib/gitlab/ci/pipeline/expression/lexeme/matches.rb (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb new file mode 100644 index 00000000000..806f2082227 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Matches < Lexeme::Operator + PATTERN = /=~/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + text = @left.evaluate(variables) + regexp = @right.evaluate(variables) + + regexp.scan(text).any? + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end -- cgit v1.2.1 From 475d2edf04c13c6a5ed2b5c68d75efd2a8024c2b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 15 May 2018 12:37:09 +0200 Subject: Reorganize exceptions in pipeline expressions module --- lib/gitlab/ci/pipeline/expression.rb | 10 ++++++++++ lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb | 3 +-- lib/gitlab/ci/pipeline/expression/lexer.rb | 4 ++-- lib/gitlab/ci/pipeline/expression/parser.rb | 2 -- lib/gitlab/ci/pipeline/expression/statement.rb | 6 +++--- 5 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 lib/gitlab/ci/pipeline/expression.rb (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb new file mode 100644 index 00000000000..f57df7c5637 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression.rb @@ -0,0 +1,10 @@ +module Gitlab + module Ci + module Pipeline + module Expression + ExpressionError = Class.new(StandardError) + RuntimeError = Class.new(ExpressionError) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 2ff527e34a8..59b8e4fad4c 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -11,10 +11,9 @@ module Gitlab end def evaluate(variables = {}) - # TODO multiline support @regexp = Gitlab::UntrustedRegexp.new(@value) rescue RegexpError - raise Parser::ParserError, 'Invalid regular expression!' + raise Expression::RuntimeError, 'Invalid regular expression!' end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index e1c68b7c3c2..ebc6565266f 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -5,6 +5,8 @@ module Gitlab class Lexer include ::Gitlab::Utils::StrongMemoize + SyntaxError = Class.new(Expression::ExpressionError) + LEXEMES = [ Expression::Lexeme::Variable, Expression::Lexeme::String, @@ -12,8 +14,6 @@ module Gitlab Expression::Lexeme::Equals ].freeze - SyntaxError = Class.new(Statement::StatementError) - MAX_TOKENS = 100 def initialize(statement, max_tokens: MAX_TOKENS) diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index fe23ab0b2f8..90f94d0b763 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -3,8 +3,6 @@ module Gitlab module Pipeline module Expression class Parser - ParserError = Class.new(Statement::StatementError) - def initialize(tokens) @tokens = tokens.to_enum @nodes = [] diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 363e0b708a6..de37c50d2a2 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -3,7 +3,7 @@ module Gitlab module Pipeline module Expression class Statement - StatementError = Class.new(StandardError) + StatementError = Class.new(Expression::ExpressionError) GRAMMAR = [ %w[variable equals string], @@ -35,13 +35,13 @@ module Gitlab def truthful? evaluate.present? - rescue StatementError + rescue Expression::ExpressionError false end def valid? parse_tree.is_a?(Lexeme::Base) - rescue StatementError + rescue Expression::ExpressionError false end end -- cgit v1.2.1 From 65f4e7b2a1fe8946109c7aa0d59999bfaaeba257 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 15 May 2018 13:04:18 +0200 Subject: Add support for pattern matching in variables expressions --- lib/gitlab/ci/pipeline/expression/lexeme/matches.rb | 2 +- lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb | 2 ++ lib/gitlab/ci/pipeline/expression/lexer.rb | 4 +++- lib/gitlab/ci/pipeline/expression/statement.rb | 4 +++- 4 files changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 806f2082227..10957598f76 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -15,7 +15,7 @@ module Gitlab text = @left.evaluate(variables) regexp = @right.evaluate(variables) - regexp.scan(text).any? + regexp.scan(text.to_s).any? end def self.build(_value, behind, ahead) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 59b8e4fad4c..f7b12914249 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -3,6 +3,8 @@ module Gitlab module Pipeline module Expression module Lexeme + require_dependency 're2' + class Pattern < Lexeme::Value PATTERN = %r{/(?.+)/}.freeze diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index ebc6565266f..4cacb1e62c9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -10,8 +10,10 @@ module Gitlab LEXEMES = [ Expression::Lexeme::Variable, Expression::Lexeme::String, + Expression::Lexeme::Pattern, Expression::Lexeme::Null, - Expression::Lexeme::Equals + Expression::Lexeme::Equals, + Expression::Lexeme::Matches ].freeze MAX_TOKENS = 100 diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index de37c50d2a2..8886cbae516 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -6,12 +6,14 @@ module Gitlab StatementError = Class.new(Expression::ExpressionError) GRAMMAR = [ + %w[variable], %w[variable equals string], %w[variable equals variable], %w[variable equals null], %w[string equals variable], %w[null equals variable], - %w[variable] + %w[variable matches pattern], + %w[pattern matches variable] ].freeze def initialize(statement, variables = {}) -- cgit v1.2.1 From df91580cc400897c5bf69d5e496b2e4db2f75e4f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 15 May 2018 13:31:45 +0200 Subject: Do not support inverse variable pattern matching --- lib/gitlab/ci/pipeline/expression/statement.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 8886cbae516..b36f1e0f865 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -12,8 +12,7 @@ module Gitlab %w[variable equals null], %w[string equals variable], %w[null equals variable], - %w[variable matches pattern], - %w[pattern matches variable] + %w[variable matches pattern] ].freeze def initialize(statement, variables = {}) -- cgit v1.2.1 From c1377c6cf0647cccd04bc6fb65d745a2e446f827 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 15 May 2018 14:35:12 +0200 Subject: Remove useless assignment in pattern lexeme --- lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index f7b12914249..62927441035 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -13,7 +13,7 @@ module Gitlab end def evaluate(variables = {}) - @regexp = Gitlab::UntrustedRegexp.new(@value) + Gitlab::UntrustedRegexp.new(@value) rescue RegexpError raise Expression::RuntimeError, 'Invalid regular expression!' end -- cgit v1.2.1 From 0ce63efe966840edb6e6184cf1abcef272a24dfc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 17 May 2018 12:29:47 +0200 Subject: Add extended /regexp/ scheme support to untrusted regexp --- lib/gitlab/untrusted_regexp.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index ce1cf737663..70d1a7c6535 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -9,6 +9,8 @@ module Gitlab # there is a strict limit on total execution time. See the RE2 documentation # at https://github.com/google/re2/wiki/Syntax for more details. class UntrustedRegexp + require_dependency 're2' + delegate :===, :source, to: :regexp def initialize(pattern, multiline: false) @@ -52,6 +54,27 @@ module Gitlab Regexp.new(pattern) end + def self.valid?(pattern) + self.fabricate(pattern) + rescue RegexpError + false + end + + def self.fabricate(pattern) + matches = pattern.match(%r{^/(?.+)/(?[ismU]*)$}) + + if matches + expression = matches[:regexp] + flags = matches[:flags] + + expression.prepend("(?#{flags})") if flags.present? + + self.new(expression, multiline: false) + else + self.new(pattern, multiline: false) + end + end + private attr_reader :regexp -- cgit v1.2.1 From a1f1e08670a7f8bd5499e16c778be16106210a44 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 17 May 2018 12:35:20 +0200 Subject: Add anti-corruption layer above expressions pattern matching --- lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 62927441035..53fb5f769d8 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -6,14 +6,14 @@ module Gitlab require_dependency 're2' class Pattern < Lexeme::Value - PATTERN = %r{/(?.+)/}.freeze + PATTERN = %r{^(?/.+/[ismU]*)$}.freeze def initialize(regexp) @value = regexp end def evaluate(variables = {}) - Gitlab::UntrustedRegexp.new(@value) + Gitlab::UntrustedRegexp.fabricate(@value) rescue RegexpError raise Expression::RuntimeError, 'Invalid regular expression!' end -- cgit v1.2.1 From 8b3e21b66b734b38e88f63727ee77b978ea21bfc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 17 May 2018 12:44:46 +0200 Subject: Add variables expression pattern validation support --- lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 53fb5f769d8..70a221010f3 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -10,6 +10,10 @@ module Gitlab def initialize(regexp) @value = regexp + + unless Gitlab::UntrustedRegexp.valid?(@value) + raise Lexer::SyntaxError, 'Invalid regular expression!' + end end def evaluate(variables = {}) -- cgit v1.2.1 From 19428e800895ba20eacb3357285acef8d69f6d8c Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 7 May 2018 18:22:07 +0200 Subject: Preload pipeline data for project pipelines When displaying the pipelines of a project we now preload the following data: 1. Authors of the commits that belong to these pipelines 2. The number of warnings per pipeline, which is used by Ci::Pipeline#has_warnings? == Commit Authors Previously this data was queried for every Commit separately, leading to 20 SQL queries being executed in the worst case. With an average of 3 to 5 milliseconds per SQL query this could result in 100 milliseconds being spent in _just_ getting Commit authors. To preload this data Commit#author now uses BatchLoader (through Commit#lazy_author), and a separate module Gitlab::Ci::Pipeline::Preloader is used to ensure all authors are loaded before they are used. == Number of warnings This changes Ci::Pipeline#has_warnings? so it supports preloading of the number of warnings per pipeline. This removes the need for executing a COUNT(*) query for every pipeline just to see if it has any warnings or not. --- lib/gitlab/ci/pipeline/preloader.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lib/gitlab/ci/pipeline/preloader.rb (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb new file mode 100644 index 00000000000..e7a2e5511cf --- /dev/null +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + # Class for preloading data associated with pipelines such as commit + # authors. + module Preloader + def self.preload(pipelines) + # This ensures that all the pipeline commits are eager loaded before we + # start using them. + pipelines.each(&:commit) + + pipelines.each do |pipeline| + # This preloads the author of every commit. We're using "lazy_author" + # here since "author" immediately loads the data on the first call. + pipeline.commit.try(:lazy_author) + + # This preloads the number of warnings for every pipeline, ensuring + # that Ci::Pipeline#has_warnings? doesn't execute any additional + # queries. + pipeline.number_of_warnings + end + end + end + end + end +end -- cgit v1.2.1 From b11c218ad9d4635c2230e7f8d105236ca32e5072 Mon Sep 17 00:00:00 2001 From: Harish Ved Date: Thu, 17 May 2018 16:20:41 +0000 Subject: Fix: Use case in-sensitive ordering by name for groups --- lib/tasks/migrate/setup_postgresql.rake | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index af30ecb0e9b..e7aab50e42a 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -8,6 +8,7 @@ task setup_postgresql: :environment do require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') + require Rails.root.join('db/migrate/20180504195842_project_name_lower_index.rb') require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb') NamespacesProjectsPathLowerIndexes.new.up @@ -18,5 +19,6 @@ task setup_postgresql: :environment do IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up UsersNameLowerIndex.new.up + ProjectNameLowerIndex.new.up AddPathIndexToRedirectRoutes.new.up end -- cgit v1.2.1 From 63c58a6dd0d8a4b3db172ffe6d0e32e127cdabd1 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 17 May 2018 21:20:15 +0200 Subject: Memoize Gitlab::Database.version This removes the need for running a database query every time we want to check the database version. --- lib/gitlab/database.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 76501dd50e8..d49d055c3f2 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -43,7 +43,7 @@ module Gitlab end def self.version - database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] + @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end def self.join_lateral_supported? -- cgit v1.2.1 From 6c190d273d18d21e50dea65645185839bf067714 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 18 May 2018 01:05:11 +0000 Subject: Move API group deletion to Sidekiq --- lib/api/groups.rb | 4 +++- lib/api/v3/groups.rb | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 0d125cd7831..03b6b30a0d8 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -167,8 +167,10 @@ module API Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/46285') destroy_conditionally!(group) do |group| - ::Groups::DestroyService.new(group, current_user).execute + ::Groups::DestroyService.new(group, current_user).async_execute end + + accepted! end desc 'Get a list of projects in this group.' do diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb index 3844fd4810d..4fa7d196e50 100644 --- a/lib/api/v3/groups.rb +++ b/lib/api/v3/groups.rb @@ -131,8 +131,9 @@ module API delete ":id" do group = find_group!(params[:id]) authorize! :admin_group, group - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/46285') - present ::Groups::DestroyService.new(group, current_user).execute, with: Entities::GroupDetail, current_user: current_user + ::Groups::DestroyService.new(group, current_user).async_execute + + accepted! end desc 'Get a list of projects in this group.' do -- cgit v1.2.1 From 6b98033d9120c7dfc5276f623e27e7af22dd7b88 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 17 May 2018 23:48:59 -0700 Subject: Fix api_json.log not always reporting the right HTTP status code As described in https://github.com/aserafin/grape_logging/issues/45, if a Grape error is caught by the handlers and a different return code is returned, then the api_json.log would have a 500 error code instead of the right value. Inserting the GrapeLogging middleware after the Grape middleware fixes this problem. Seen in https://gitlab.com/gitlab-com/infrastructure/issues/4249 --- lib/api/api.rb | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 5139e869c71..2fbeaaffcfe 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -8,14 +8,15 @@ module API PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze - use GrapeLogging::Middleware::RequestLogger, - logger: Logger.new(LOG_FILENAME), - formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, - include: [ - GrapeLogging::Loggers::FilterParameters.new, - GrapeLogging::Loggers::ClientEnv.new, - Gitlab::GrapeLogging::Loggers::UserLogger.new - ] + insert_before Grape::Middleware::Error, + GrapeLogging::Middleware::RequestLogger, + logger: Logger.new(LOG_FILENAME), + formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, + include: [ + GrapeLogging::Loggers::FilterParameters.new, + GrapeLogging::Loggers::ClientEnv.new, + Gitlab::GrapeLogging::Loggers::UserLogger.new + ] allow_access_with_scope :api prefix :api -- cgit v1.2.1 From af9b0bfbae84a402e5c706ac29772b0d70dfa156 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 May 2018 10:14:10 +0200 Subject: Simplify untrusted regexp factory method --- lib/gitlab/untrusted_regexp.rb | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 70d1a7c6535..dc2d91dfa23 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -55,7 +55,7 @@ module Gitlab end def self.valid?(pattern) - self.fabricate(pattern) + !!self.fabricate(pattern) rescue RegexpError false end @@ -63,16 +63,13 @@ module Gitlab def self.fabricate(pattern) matches = pattern.match(%r{^/(?.+)/(?[ismU]*)$}) - if matches - expression = matches[:regexp] - flags = matches[:flags] + raise RegexpError, 'Invalid regular expression!' if matches.nil? - expression.prepend("(?#{flags})") if flags.present? + expression = matches[:regexp] + flags = matches[:flags] + expression.prepend("(?#{flags})") if flags.present? - self.new(expression, multiline: false) - else - self.new(pattern, multiline: false) - end + self.new(expression, multiline: false) end private -- cgit v1.2.1 From afa245142117a7e90ff6046133a2402fb8c09cb1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 May 2018 10:14:19 +0200 Subject: Simplify pattern lexeme fabrication and matcher --- lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 70a221010f3..9b239c29ea4 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -6,7 +6,7 @@ module Gitlab require_dependency 're2' class Pattern < Lexeme::Value - PATTERN = %r{^(?/.+/[ismU]*)$}.freeze + PATTERN = %r{^/.+/[ismU]*$}.freeze def initialize(regexp) @value = regexp @@ -23,7 +23,7 @@ module Gitlab end def self.build(string) - new(string.match(PATTERN)[:regexp]) + new(string) end end end -- cgit v1.2.1 From c0e77f7c9cc24104981bb8f6973ceeb9c311e1e2 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Fri, 18 May 2018 10:25:59 +0000 Subject: Resolve "Expand API: Render an arbitrary Markdown document" --- lib/api/api.rb | 1 + lib/api/markdown.rb | 33 +++++++++++++++++++++++++++++++++ lib/banzai/filter/reference_filter.rb | 2 +- lib/banzai/pipeline/gfm_pipeline.rb | 4 ++-- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 lib/api/markdown.rb (limited to 'lib') diff --git a/lib/api/api.rb b/lib/api/api.rb index 2fbeaaffcfe..de20b2b8e67 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -140,6 +140,7 @@ module API mount ::API::Keys mount ::API::Labels mount ::API::Lint + mount ::API::Markdown mount ::API::Members mount ::API::MergeRequestDiffs mount ::API::MergeRequests diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb new file mode 100644 index 00000000000..b9ed68aa584 --- /dev/null +++ b/lib/api/markdown.rb @@ -0,0 +1,33 @@ +module API + class Markdown < Grape::API + params do + requires :text, type: String, desc: "The markdown text to render" + optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown" + optional :project, type: String, desc: "The full path of a project to use as the context when creating references using GitLab Flavored Markdown" + end + resource :markdown do + desc "Render markdown text" do + detail "This feature was introduced in GitLab 11.0." + end + post do + # Explicitly set CommonMark as markdown engine to use. + # Remove this set when https://gitlab.com/gitlab-org/gitlab-ce/issues/43011 is done. + context = { markdown_engine: :common_mark, only_path: false } + + if params[:project] + project = Project.find_by_full_path(params[:project]) + + not_found!("Project") unless can?(current_user, :read_project, project) + + context[:project] = project + else + context[:skip_project_check] = true + end + + context[:pipeline] = params[:gfm] ? :full : :plain_markdown + + { html: Banzai.render(params[:text], context) } + end + end + end +end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index b9d5ecf70ec..2f023f4f242 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -73,7 +73,7 @@ module Banzai # # Note that while the key might exist, its value could be nil! def validate - needs :project + needs :project unless skip_project_check? end # Iterates over all and text() nodes in a document. diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8b2f05fffec..a1f24e8b093 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -42,9 +42,9 @@ module Banzai end def self.transform_context(context) - context.merge( - only_path: true, + context[:only_path] = true unless context.key?(:only_path) + context.merge( # EmojiFilter asset_host: Gitlab::Application.config.asset_host, asset_root: Gitlab.config.gitlab.base_url -- cgit v1.2.1 From 18a8eb96b34db63101c2b1210c1f93342b138a55 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Fri, 11 May 2018 16:44:38 +0200 Subject: Calculating repository checksums executed by Gitaly OPT_OUT status has been removed, and alternative implementation removed. Also checks if the repository exists before executing the checksum RPC to guard against NotFound errors. Closes gitlab-org/gitaly#1105 --- lib/gitlab/git/repository.rb | 44 ++++++-------------------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 25487f53999..f612d9dc597 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1576,14 +1576,12 @@ module Gitlab end def checksum - gitaly_migrate(:calculate_checksum, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.calculate_checksum - else - calculate_checksum_by_shelling_out - end - end + # The exists? RPC is much cheaper, so we perform this request first + raise NoRepository, "Repository does not exists" unless exists? + + gitaly_repository_client.calculate_checksum + rescue GRPC::NotFound + raise NoRepository # Guard against data races. end private @@ -2498,36 +2496,6 @@ module Gitlab rev_parse_target(ref).oid end - def calculate_checksum_by_shelling_out - raise NoRepository unless exists? - - args = %W(--git-dir=#{path} show-ref --heads --tags) - output, status = run_git(args) - - if status.nil? || !status.zero? - # Non-valid git repositories return 128 as the status code and an error output - raise InvalidRepository if status == 128 && output.to_s.downcase =~ /not a git repository/ - # Empty repositories returns with a non-zero status and an empty output. - raise ChecksumError, output unless output.blank? - - return EMPTY_REPOSITORY_CHECKSUM - end - - refs = output.split("\n") - - result = refs.inject(nil) do |checksum, ref| - value = Digest::SHA1.hexdigest(ref).hex - - if checksum.nil? - value - else - checksum ^ value - end - end - - result.to_s(16) - end - def build_git_cmd(*args) object_directories = alternate_object_directories.join(File::PATH_SEPARATOR) -- cgit v1.2.1