Merge remote-tracking branch 'dev/15-1-stable' into 15-1-stable15-1-stable

17 files changed, 139 insertions, 61 deletions

diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index dedda82091f..71594025688 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -142,7 +142,7 @@ module API
             Gitlab::UsageDataCounters::EditorUniqueCounter.track_web_ide_edit_action(author: current_user)
           end
 
-          present commit_detail, with: Entities::CommitDetail, stats: params[:stats]
+          present commit_detail, with: Entities::CommitDetail, include_stats: params[:stats], current_user: current_user
         else
           render_api_error!(result[:message], 400)
         end
@@ -161,7 +161,7 @@ module API
 
         not_found! 'Commit' unless commit
 
-        present commit, with: Entities::CommitDetail, stats: params[:stats], current_user: current_user
+        present commit, with: Entities::CommitDetail, include_stats: params[:stats], current_user: current_user
       end
 
       desc 'Get the diff for a specific commit of a project' do
diff --git a/lib/api/entities/commit.rb b/lib/api/entities/commit.rb
index fd23c23b980..6cd180cd584 100644
--- a/lib/api/entities/commit.rb
+++ b/lib/api/entities/commit.rb
@@ -12,7 +12,9 @@ module API
       expose :trailers
 
       expose :web_url do |commit, _options|
-        Gitlab::UrlBuilder.build(commit)
+        c = commit
+        c = c.__subject__ if c.is_a?(Gitlab::View::Presenter::Base)
+        Gitlab::UrlBuilder.build(c)
       end
     end
   end
diff --git a/lib/api/entities/commit_detail.rb b/lib/api/entities/commit_detail.rb
index 61238102e9d..cc529639359 100644
--- a/lib/api/entities/commit_detail.rb
+++ b/lib/api/entities/commit_detail.rb
@@ -3,8 +3,10 @@
 module API
   module Entities
     class CommitDetail < Commit
-      expose :stats, using: Entities::CommitStats, if: :stats
-      expose :status
+      include ::API::Helpers::Presentable
+
+      expose :stats, using: Entities::CommitStats, if: :include_stats
+      expose :status_for, as: :status
       expose :project_id
 
       expose :last_pipeline do |commit, options|
diff --git a/lib/api/helpers/packages/basic_auth_helpers.rb b/lib/api/helpers/packages/basic_auth_helpers.rb
index 6c381d85cd8..ebedb3b7563 100644
--- a/lib/api/helpers/packages/basic_auth_helpers.rb
+++ b/lib/api/helpers/packages/basic_auth_helpers.rb
@@ -14,28 +14,12 @@ module API
         include Constants
         include Gitlab::Utils::StrongMemoize
 
-        def unauthorized_user_project
-          @unauthorized_user_project ||= find_project(params[:id])
-        end
-
-        def unauthorized_user_project!
-          unauthorized_user_project || not_found!
-        end
-
-        def unauthorized_user_group
-          @unauthorized_user_group ||= find_group(params[:id])
-        end
-
-        def unauthorized_user_group!
-          unauthorized_user_group || not_found!
-        end
-
         def authorized_user_project
           @authorized_user_project ||= authorized_project_find!
         end
 
         def authorized_project_find!
-          project = unauthorized_user_project
+          project = find_project(params[:id])
 
           unless project && can?(current_user, :read_project, project)
             return unauthorized_or! { not_found! }
diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb
index 5bf3c3b8aac..b6ed4103d2b 100644
--- a/lib/api/pypi_packages.rb
+++ b/lib/api/pypi_packages.rb
@@ -84,6 +84,16 @@ module API
 
         body content
       end
+
+      def ensure_group!
+        find_group(params[:id]) || not_found!
+        find_authorized_group!
+      end
+
+      def ensure_project!
+        find_project(params[:id]) || not_found!
+        authorized_user_project
+      end
     end
 
     params do
@@ -91,7 +101,7 @@ module API
     end
     resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       after_validation do
-        unauthorized_user_group!
+        ensure_group!
       end
 
       namespace ':id/-/packages/pypi' do
@@ -101,7 +111,8 @@ module API
 
         route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
         get 'files/:sha256/*file_identifier' do
-          group = unauthorized_user_group!
+          group = find_authorized_group!
+          authorize_read_package!(group)
 
           filename = "#{params[:file_identifier]}.#{params[:format]}"
           package = Packages::Pypi::PackageFinder.new(current_user, group, { filename: filename, sha256: params[:sha256] }).execute
@@ -146,7 +157,7 @@ module API
 
     resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       before do
-        unauthorized_user_project!
+        ensure_project!
       end
 
       namespace ':id/packages/pypi' do
@@ -160,7 +171,8 @@ module API
 
         route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
         get 'files/:sha256/*file_identifier' do
-          project = unauthorized_user_project!
+          project = authorized_user_project
+          authorize_read_package!(project)
 
           filename = "#{params[:file_identifier]}.#{params[:format]}"
           package = Packages::Pypi::PackageFinder.new(current_user, project, { filename: filename, sha256: params[:sha256] }).execute
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 2e21f591667..023c993d1d6 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -183,7 +183,7 @@ module API
           compare = CompareService.new(user_project, params[:to]).execute(target_project, params[:from], straight: params[:straight])
 
           if compare
-            present compare, with: Entities::Compare
+            present compare, with: Entities::Compare, current_user: current_user
           else
             not_found!("Ref")
           end
diff --git a/lib/api/search.rb b/lib/api/search.rb
index fd4d46cf77d..d8789d8839e 100644
--- a/lib/api/search.rb
+++ b/lib/api/search.rb
@@ -102,7 +102,7 @@ module API
       get do
         verify_search_scope!(resource: nil)
 
-        present search, with: entity
+        present search, with: entity, current_user: current_user
       end
     end
 
@@ -124,7 +124,7 @@ module API
       get ':id/(-/)search' do
         verify_search_scope!(resource: user_group)
 
-        present search(group_id: user_group.id), with: entity
+        present search(group_id: user_group.id), with: entity, current_user: current_user
       end
     end
 
@@ -145,7 +145,7 @@ module API
         use :pagination
       end
       get ':id/(-/)search' do
-        present search({ project_id: user_project.id, repository_ref: params[:ref] }), with: entity
+        present search({ project_id: user_project.id, repository_ref: params[:ref] }), with: entity, current_user: current_user
       end
     end
   end
diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb
index 5c71a18c6d0..2b51ab91c40 100644
--- a/lib/api/submodules.rb
+++ b/lib/api/submodules.rb
@@ -39,7 +39,7 @@ module API
 
         if result[:status] == :success
           commit_detail = user_project.repository.commit(result[:result])
-          present commit_detail, with: Entities::CommitDetail
+          present commit_detail, with: Entities::CommitDetail, current_user: current_user
         else
           render_api_error!(result[:message], result[:http_status] || 400)
         end
diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb
index a615abc1989..817bea42757 100644
--- a/lib/banzai/filter/commit_trailers_filter.rb
+++ b/lib/banzai/filter/commit_trailers_filter.rb
@@ -17,21 +17,10 @@ module Banzai
       include ActionView::Helpers::TagHelper
       include AvatarsHelper
 
-      TRAILER_REGEXP = /(?<label>[[:alpha:]-]+-by:)/i.freeze
-      AUTHOR_REGEXP = /(?<author_name>.+)/.freeze
-      # Devise.email_regexp wouldn't work here since its designed to match
-      # against strings that only contains email addresses; the \A and \z
-      # around the expression will only match if the string being matched
-      # contains just the email nothing else.
-      MAIL_REGEXP = /&lt;(?<author_email>[^@\s]+@[^@\s]+)&gt;/.freeze
-      FILTER_REGEXP = /(?<trailer>^\s*#{TRAILER_REGEXP}\s*#{AUTHOR_REGEXP}\s+#{MAIL_REGEXP}$)/mi.freeze
-
       def call
         doc.xpath('descendant-or-self::text()').each do |node|
           content = node.to_html
 
-          next unless content.match(FILTER_REGEXP)
-
           html = trailer_filter(content)
 
           next if html == content
@@ -52,11 +41,24 @@ module Banzai
       # Returns a String with all trailer lines replaced with links to GitLab
       # users and mailto links to non GitLab users. All links have `data-trailer`
       # and `data-user` attributes attached.
+      #
+      # The code intentionally avoids using Regex for security and performance
+      # reasons: https://gitlab.com/gitlab-org/gitlab/-/issues/363734
       def trailer_filter(text)
-        text.gsub(FILTER_REGEXP) do |author_match|
-          label = $~[:label]
-          "#{label} #{parse_user($~[:author_name], $~[:author_email], label)}"
-        end
+        text.lines.map! do |line|
+          trailer, rest = line.split(':', 2)
+
+          next line unless trailer.downcase.end_with?('-by') && rest.present?
+
+          chunks = rest.split
+          author_email = chunks.pop.delete_prefix('&lt;').delete_suffix('&gt;')
+          next line unless Devise.email_regexp.match(author_email)
+
+          author_name = chunks.join(' ').strip
+          trailer = "#{trailer.strip}:"
+
+          "#{trailer} #{link_to_user_or_email(author_name, author_email, trailer)}\n"
+        end.join
       end
 
       # Find a GitLab user using the supplied email and generate
@@ -67,7 +69,7 @@ module Banzai
       # trailer - String trailer used in the commit message
       #
       # Returns a String with a link to the user.
-      def parse_user(name, email, trailer)
+      def link_to_user_or_email(name, email, trailer)
         link_to_user User.find_by_any_email(email),
           name: name,
           email: email,
diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb
index 60881b5f511..262c0b5340d 100644
--- a/lib/banzai/filter/image_link_filter.rb
+++ b/lib/banzai/filter/image_link_filter.rb
@@ -34,17 +34,20 @@ module Banzai
             img.remove_attribute('data-diagram-src')
           end
 
-          link.children = if link_replaces_image
-                            img['alt'] || img['data-src'] || img['src']
-                          else
-                            img.clone
-                          end
+          link.children = link_replaces_image ? link_children(img) : img.clone
 
           img.replace(link)
         end
 
         doc
       end
+
+      private
+
+      def link_children(img)
+        [img['alt'], img['data-src'], img['src']]
+          .map { |f| Sanitize.fragment(f).presence }.compact.first || ''
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/pathological_markdown_filter.rb b/lib/banzai/filter/pathological_markdown_filter.rb
new file mode 100644
index 00000000000..0f94150c7a1
--- /dev/null
+++ b/lib/banzai/filter/pathological_markdown_filter.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    class PathologicalMarkdownFilter < HTML::Pipeline::TextFilter
+      # It's not necessary for this to be precise - we just need to detect
+      # when there are a non-trivial number of unclosed image links.
+      # So we don't really care about code blocks, etc.
+      # See https://gitlab.com/gitlab-org/gitlab/-/issues/370428
+      REGEX = /!\[(?:[^\]])+?!\[/.freeze
+      DETECTION_MAX = 10
+
+      def call
+        count = 0
+
+        @text.scan(REGEX) do |_match|
+          count += 1
+          break if count > DETECTION_MAX
+        end
+
+        return @text if count <= DETECTION_MAX
+
+        "_Unable to render markdown - too many unclosed markdown image links detected._"
+      end
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/incident_management/timeline_event_pipeline.rb b/lib/banzai/pipeline/incident_management/timeline_event_pipeline.rb
index 01ee3f5d9e8..eef2b2674dd 100644
--- a/lib/banzai/pipeline/incident_management/timeline_event_pipeline.rb
+++ b/lib/banzai/pipeline/incident_management/timeline_event_pipeline.rb
@@ -11,9 +11,9 @@ module Banzai
         def self.filters
           @filters ||= FilterArray[
             *super,
+            Filter::SanitizationFilter,
             *Banzai::Pipeline::GfmPipeline.reference_filters,
             Filter::EmojiFilter,
-            Filter::SanitizationFilter,
             Filter::ExternalLinkFilter,
             Filter::ImageLinkFilter
           ]
diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb
index 1da0f72996b..fb6f6e9077d 100644
--- a/lib/banzai/pipeline/plain_markdown_pipeline.rb
+++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb
@@ -5,6 +5,7 @@ module Banzai
     class PlainMarkdownPipeline < BasePipeline
       def self.filters
         FilterArray[
+          Filter::PathologicalMarkdownFilter,
           Filter::MarkdownPreEscapeFilter,
           Filter::MarkdownFilter,
           Filter::MarkdownPostEscapeFilter
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 9fb34f74c82..529d4354a1b 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -5,6 +5,8 @@ module Gitlab
     class CommitService
       include Gitlab::EncodingHelper
 
+      TREE_ENTRIES_DEFAULT_LIMIT = 1000
+
       def initialize(repository)
         @gitaly_repo = repository.gitaly_repository
         @repository = repository
@@ -112,6 +114,9 @@ module Gitlab
       end
 
       def tree_entries(repository, revision, path, recursive, pagination_params)
+        pagination_params ||= {}
+        pagination_params[:limit] ||= TREE_ENTRIES_DEFAULT_LIMIT
+
         request = Gitaly::GetTreeEntriesRequest.new(
           repository: @gitaly_repo,
           revision: encode_binary(revision),
diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb
index 09ba95666de..f426f70800c 100644
--- a/lib/gitlab/markdown_cache.rb
+++ b/lib/gitlab/markdown_cache.rb
@@ -11,7 +11,7 @@ module Gitlab
     # this if the change to the renderer output is a new feature or a
     # minor bug fix.
     # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330313
-    CACHE_COMMONMARK_VERSION       = 31
+    CACHE_COMMONMARK_VERSION       = 32
     CACHE_COMMONMARK_VERSION_START = 10
 
     BaseError = Class.new(StandardError)
diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb
index feb2c3c1d7d..896e7e3f65e 100644
--- a/lib/gitlab/set_cache.rb
+++ b/lib/gitlab/set_cache.rb
@@ -68,6 +68,10 @@ module Gitlab
       with { |redis| redis.ttl(cache_key(key)) }
     end
 
+    def count(key)
+      with { |redis| redis.scard(cache_key(key)) }
+    end
+
     private
 
     def with(&blk)
diff --git a/lib/gitlab/zentao/client.rb b/lib/gitlab/zentao/client.rb
index 4da4631eecf..0de4ca3a09b 100644
--- a/lib/gitlab/zentao/client.rb
+++ b/lib/gitlab/zentao/client.rb
@@ -5,6 +5,10 @@ module Gitlab
     class Client
       Error = Class.new(StandardError)
       ConfigError = Class.new(Error)
+      RequestError = Class.new(Error)
+
+      CACHE_MAX_SET_SIZE = 5_000
+      CACHE_TTL = 1.month.freeze
 
       attr_reader :integration
 
@@ -29,11 +33,21 @@ module Gitlab
       end
 
       def fetch_issues(params = {})
-        get("products/#{zentao_product_xid}/issues", params)
+        get("products/#{zentao_product_xid}/issues", params).tap do |response|
+          mark_issues_as_seen_in_product(response['issues'])
+        end
       end
 
       def fetch_issue(issue_id)
-        raise Gitlab::Zentao::Client::Error, 'invalid issue id' unless issue_id_pattern.match(issue_id)
+        raise Error, 'invalid issue id' unless issue_id_pattern.match(issue_id)
+
+        # Only return issues that are associated with the product configured in
+        # the integration. Due to a lack of available data in the ZenTao APIs, we
+        # can only determine if an issue belongs to a product if the issue was
+        # previously returned in the `#fetch_issues` call.
+        #
+        # See https://gitlab.com/gitlab-org/gitlab/-/issues/360372#note_1016963713
+        raise RequestError unless issue_seen_in_product?(issue_id)
 
         get("issues/#{issue_id}")
       end
@@ -48,17 +62,15 @@ module Gitlab
         options = { headers: headers, query: params }
         response = Gitlab::HTTP.get(url(path), options)
 
-        raise Gitlab::Zentao::Client::Error, 'request error' unless response.success?
+        raise RequestError unless response.success?
 
         Gitlab::Json.parse(response.body)
       rescue JSON::ParserError
-        raise Gitlab::Zentao::Client::Error, 'invalid response format'
+        raise Error, 'invalid response format'
       end
 
       def url(path)
-        host = integration.api_url.presence || integration.url
-
-        URI.parse(Gitlab::Utils.append_path(host, "api.php/v1/#{path}"))
+        URI.parse(Gitlab::Utils.append_path(integration.client_url, "api.php/v1/#{path}"))
       end
 
       def headers
@@ -71,6 +83,30 @@ module Gitlab
       def zentao_product_xid
         integration.zentao_product_xid
       end
+
+      def issue_ids_cache_key
+        @issue_ids_cache_key ||= [
+          :zentao_product_issues,
+          OpenSSL::Digest::SHA256.hexdigest(integration.client_url),
+          zentao_product_xid
+        ].join(':')
+      end
+
+      def issue_ids_cache
+        @issue_ids_cache ||= ::Gitlab::SetCache.new(expires_in: CACHE_TTL)
+      end
+
+      def mark_issues_as_seen_in_product(issues)
+        return unless issues && issue_ids_cache.count(issue_ids_cache_key) < CACHE_MAX_SET_SIZE
+
+        ids = issues.map { _1['id'] }
+
+        issue_ids_cache.write(issue_ids_cache_key, ids)
+      end
+
+      def issue_seen_in_product?(id)
+        issue_ids_cache.include?(issue_ids_cache_key, id)
+      end
     end
   end
 end