diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-02 12:18:53 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-02 12:18:53 +0000 |
commit | 51d59a3538b97d85ebb46039044d3f498809b55a (patch) | |
tree | d574af08cc1e2ef8fa8337a0850fa36a7b9ab527 /lib | |
parent | 74da249f7e22c20e144ba3c044c6bdeb5df86cd4 (diff) | |
download | gitlab-ce-51d59a3538b97d85ebb46039044d3f498809b55a.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/maven_packages.rb | 12 | ||||
-rw-r--r-- | lib/banzai/filter/base_sanitization_filter.rb | 3 | ||||
-rw-r--r-- | lib/banzai/filter/code_language_filter.rb | 71 | ||||
-rw-r--r-- | lib/banzai/filter/kroki_filter.rb | 6 | ||||
-rw-r--r-- | lib/banzai/filter/math_filter.rb | 2 | ||||
-rw-r--r-- | lib/banzai/filter/mermaid_filter.rb | 2 | ||||
-rw-r--r-- | lib/banzai/filter/plantuml_filter.rb | 2 | ||||
-rw-r--r-- | lib/banzai/filter/syntax_highlight_filter.rb | 51 | ||||
-rw-r--r-- | lib/banzai/pipeline/ascii_doc_pipeline.rb | 1 | ||||
-rw-r--r-- | lib/banzai/pipeline/gfm_pipeline.rb | 1 | ||||
-rw-r--r-- | lib/banzai/pipeline/markup_pipeline.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/color.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/database_importers/self_monitoring/helpers.rb | 25 | ||||
-rw-r--r-- | lib/gitlab/database_importers/self_monitoring/project/create_service.rb | 171 | ||||
-rw-r--r-- | lib/gitlab/database_importers/self_monitoring/project/delete_service.rb | 46 |
15 files changed, 133 insertions, 300 deletions
diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index a0d2fe45813..ed3479fce2d 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -319,7 +319,7 @@ module API end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true put ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do - unprocessable_entity! if Gitlab::FIPS.enabled? && params['file.md5'] + unprocessable_entity! if Gitlab::FIPS.enabled? && params[:file].md5 authorize_upload! bad_request!('File is too large') if user_project.actual_limits.exceeded?(:maven_max_file_size, params[:file].size) @@ -347,16 +347,12 @@ module API else file_params = { file: params[:file], - size: params['file.size'], + size: params[:file].size, file_name: file_name, - file_sha1: params['file.sha1'], - file_md5: params['file.md5'] + file_sha1: params[:file].sha1, + file_md5: params[:file].md5 } - if Feature.enabled?(:read_fingerprints_from_uploaded_file_in_maven_upload, user_project) - file_params.merge!(size: params[:file].size, file_sha1: params[:file].sha1, file_md5: params[:file].md5) - end - ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)).execute track_package_event('push_package', :maven, project: user_project, namespace: user_project.namespace) if jar_file?(format) end diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb index 3b00d1a9824..0735fbb8d4c 100644 --- a/lib/banzai/filter/base_sanitization_filter.rb +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -25,7 +25,8 @@ module Banzai # Allow data-math-style attribute in order to support LaTeX formatting allowlist[:attributes]['code'] = %w(data-math-style) - allowlist[:attributes]['pre'] = %w(data-math-style data-mermaid-style data-kroki-style) + allowlist[:attributes]['pre'] = %w(data-canonical-lang data-lang-params + data-math-style data-mermaid-style data-kroki-style) # Allow html5 details/summary elements allowlist[:elements].push('details') diff --git a/lib/banzai/filter/code_language_filter.rb b/lib/banzai/filter/code_language_filter.rb new file mode 100644 index 00000000000..60e5a4063df --- /dev/null +++ b/lib/banzai/filter/code_language_filter.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML Filter to convert use of `lang` attribute into a common format, + # data-canonical-lang, as the `lang` attribute is really meant for accessibility + # and not for specifying code highlight language. + # See https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/lang#accessibility + # This also provides one place to transform the language specification format, whether it + # sits on the `pre` or `code`, or in a `class` or `lang` attribute + class CodeLanguageFilter < HTML::Pipeline::Filter + include OutputSafety + + LANG_PARAMS_DELIMITER = ':' + LANG_ATTR = 'data-canonical-lang' + LANG_PARAMS_ATTR = 'data-lang-params' + + CSS = 'pre > code:only-child' + XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze + + def call + doc.xpath(XPATH).each do |node| + transform_node(node) + end + + doc + end + + def transform_node(code_node) + return if code_node.parent&.parent.nil? + + lang, lang_params = parse_lang_params(code_node) + pre_node = code_node.parent + + pre_node.remove_attribute('lang') if lang.present? + pre_node.set_attribute(LANG_ATTR, escape_once(lang)) if lang.present? + pre_node.set_attribute(LANG_PARAMS_ATTR, escape_once(lang_params)) if lang_params.present? + + # cmark-gfm added this, it's now in data-lang-params + pre_node.remove_attribute('data-meta') + end + + private + + # cmark-gfm's FULL_INFO_STRING render option works with the space delimiter. + # Which means the language specified on a code block is parsed with spaces. Anything + # after the first space is placed in the `data-meta` attribute. + # However GitLab recognizes `:` as an additional delimiter on the lang attribute. + # So parse out the extra parameter. + # + # Original + # "```suggestion:+1-10 more```" -> '<pre data-canonical-lang="suggestion:+1-10" data-lang-params="more">'. + # + # With extra parsing + # "```suggestion:+1-10 more```" -> '<pre data-canonical-lang="suggestion" data-lang-params="+1-10 more">'. + def parse_lang_params(code_node) + pre_node = code_node.parent + language = pre_node.attr('lang') + + return unless language + + language, language_params = language.split(LANG_PARAMS_DELIMITER, 2) + + # cmark-gfm places extra lang parameters into data-meta + language_params = [pre_node.attr('data-meta'), language_params].compact.join(' ') + + [language, language_params] + end + end + end +end diff --git a/lib/banzai/filter/kroki_filter.rb b/lib/banzai/filter/kroki_filter.rb index 2b9e2a22c11..04f1a1b4f3c 100644 --- a/lib/banzai/filter/kroki_filter.rb +++ b/lib/banzai/filter/kroki_filter.rb @@ -18,8 +18,8 @@ module Banzai diagram_selectors = ::Gitlab::Kroki.formats(settings) .map do |diagram_type| - %(pre[lang="#{diagram_type}"] > code, - pre > code[lang="#{diagram_type}"]) + %(pre[data-canonical-lang="#{diagram_type}"] > code, + pre > code[data-canonical-lang="#{diagram_type}"]) end .join(', ') @@ -28,7 +28,7 @@ module Banzai diagram_format = "svg" doc.xpath(xpath).each do |node| - diagram_type = node.parent['lang'] || node['lang'] + diagram_type = node.parent['data-canonical-lang'] || node['data-canonical-lang'] next unless diagram_selectors.include?(diagram_type) diagram_src = node.content diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 9b6fc71077a..e568f51652f 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -12,7 +12,7 @@ module Banzai # Handle the $`...`$ and ```math syntax in this filter. # Also add necessary classes any existing math blocks. - CSS_MATH = 'pre[lang="math"] > code' + CSS_MATH = 'pre[data-canonical-lang="math"] > code' XPATH_MATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_MATH).freeze CSS_CODE = 'code' XPATH_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_CODE).freeze diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb index aaaf851ccf0..b562cbcdefe 100644 --- a/lib/banzai/filter/mermaid_filter.rb +++ b/lib/banzai/filter/mermaid_filter.rb @@ -4,7 +4,7 @@ module Banzai module Filter class MermaidFilter < HTML::Pipeline::Filter - CSS = 'pre[lang="mermaid"] > code' + CSS = 'pre[data-canonical-lang="mermaid"] > code' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze def call diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index 6a1fa64fb76..2e5f1b29c52 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -32,7 +32,7 @@ module Banzai def lang_tag @lang_tag ||= Gitlab::Utils::Nokogiri - .css_to_xpath('pre[lang="plantuml"] > code, pre > code[lang="plantuml"]').freeze + .css_to_xpath('pre[data-canonical-lang="plantuml"] > code, pre > code[data-canonical-lang="plantuml"]').freeze end def settings diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 3da6ce5c90c..e02d668a1ca 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -12,8 +12,6 @@ module Banzai class SyntaxHighlightFilter < TimeoutHtmlPipelineFilter include OutputSafety - LANG_PARAMS_DELIMITER = ':' - LANG_PARAMS_ATTR = 'data-lang-params' CSS_CLASSES = 'code highlight js-syntax-highlight' CSS = 'pre:not([data-kroki-style]) > code:only-child' @@ -27,10 +25,13 @@ module Banzai doc end - def highlight_node(node) - return if node.parent&.parent.nil? + def highlight_node(code_node) + return if code_node.parent&.parent.nil? - lang, lang_params = parse_lang_params(node) + # maintain existing attributes already added. e.g math and mermaid nodes + pre_node = code_node.parent + + lang = pre_node['data-canonical-lang'] retried = false if use_rouge?(lang) @@ -42,7 +43,7 @@ module Banzai end begin - code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code_node.text), tag: language) rescue StandardError # Gracefully handle syntax highlighter bugs/errors to ensure users can # still access an issue/comment/etc. First, retry with the plain text @@ -57,21 +58,16 @@ module Banzai retry end - # maintain existing attributes already added. e.g math and mermaid nodes - node.children = code - pre_node = node.parent + code_node.children = code # ensure there are no extra children, such as a text node that might # show up from an XSS attack - pre_node.children = node + pre_node.children = code_node - pre_node[:lang] = language pre_node.add_class(CSS_CLASSES) pre_node.add_class("language-#{language}") if language - pre_node.set_attribute('data-canonical-lang', escape_once(lang)) if lang != language - pre_node.set_attribute(LANG_PARAMS_ATTR, escape_once(lang_params)) if lang_params.present? + pre_node.set_attribute('lang', language) pre_node.set_attribute('v-pre', 'true') - pre_node.remove_attribute('data-meta') copy_code_btn = "<copy-code></copy-code>" unless language == 'suggestion' highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code">#{pre_node.to_html}#{copy_code_btn}</div>) @@ -82,33 +78,6 @@ module Banzai private - def parse_lang_params(node) - node = node.parent - - # Commonmarker's FULL_INFO_STRING render option works with the space delimiter. - # But the current behavior of GitLab's markdown renderer is different - it grabs everything as the single - # line, including language and its options. To keep backward compatibility, we have to parse the old format and - # merge with the new one. - # - # Behaviors before separating language and its parameters: - # Old ones: - # "```ruby with options```" -> '<pre><code lang="ruby with options">'. - # "```ruby:with:options```" -> '<pre><code lang="ruby:with:options">'. - # - # New ones: - # "```ruby with options```" -> '<pre><code lang="ruby" data-meta="with options">'. - # "```ruby:with:options```" -> '<pre><code lang="ruby:with:options">'. - - language = node.attr('lang') - - return unless language - - language, language_params = language.split(LANG_PARAMS_DELIMITER, 2) - language_params = [node.attr('data-meta'), language_params].compact.join(' ') - - [language, language_params] - end - # Separate method so it can be instrumented. def lex(lexer, code) lexer.lex(code) diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index 8764367426c..54306eadd41 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -6,6 +6,7 @@ module Banzai def self.filters FilterArray[ Filter::AsciiDocSanitizationFilter, + Filter::CodeLanguageFilter, Filter::AssetProxyFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 6bd9e65f431..1fd565999f5 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -11,6 +11,7 @@ module Banzai # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ + Filter::CodeLanguageFilter, Filter::PlantumlFilter, # Must always be before the SanitizationFilter to prevent XSS attacks Filter::SpacedLinkFilter, diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb index 635d4c0884e..cb421ff77b6 100644 --- a/lib/banzai/pipeline/markup_pipeline.rb +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -6,6 +6,7 @@ module Banzai def self.filters @filters ||= FilterArray[ Filter::SanitizationFilter, + Filter::CodeLanguageFilter, Filter::AssetProxyFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb index 7d9280ddba2..c31c8cb5876 100644 --- a/lib/gitlab/color.rb +++ b/lib/gitlab/color.rb @@ -9,7 +9,7 @@ module Gitlab end module Constants - DARK = Color.new('#333333') + DARK = Color.new('#1F1E24') LIGHT = Color.new('#FFFFFF') COLOR_NAME_TO_HEX = { @@ -194,8 +194,43 @@ module Gitlab PATTERN.match?(@value) end + # Implementation should match + # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L52-55 + def to_srgb(value) + normalized = value / 255.0 + normalized <= 0.03928 ? normalized / 12.92 : ((normalized + 0.055) / 1.055)**2.4 + end + + # Implementation should match + # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L57-64 + def relative_luminance(rgb) + # WCAG 2.1 formula: https://www.w3.org/TR/WCAG21/#dfn-relative-luminance + # - + # WCAG 3.0 will use APAC + # Using APAC would be the ultimate goal, but was dismissed by engineering as of now + # See https://gitlab.com/gitlab-org/gitlab-ui/-/merge_requests/3418#note_1370107090 + (0.2126 * to_srgb(rgb[0])) + (0.7152 * to_srgb(rgb[1])) + (0.0722 * to_srgb(rgb[2])) + end + + # Implementation should match + # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L66-91 def light? - valid? && rgb.sum > 500 + return false unless valid? + + luminance = relative_luminance(rgb) + light_luminance = relative_luminance([255, 255, 255]) + dark_luminance = relative_luminance([31, 30, 36]) + + contrast_light = (light_luminance + 0.05) / (luminance + 0.05) + contrast_dark = (luminance + 0.05) / (dark_luminance + 0.05) + + # Using a threshold contrast of 2.4 instead of 3 + # as this will solve weird color combinations in the mid tones + # + # Note that this is the negated condition from GitLab UI, + # because the GitLab UI implementation returns the text color, + # while this defines whether a background color is light + !(contrast_light >= 2.4 || contrast_light > contrast_dark) end def luminosity diff --git a/lib/gitlab/database_importers/self_monitoring/helpers.rb b/lib/gitlab/database_importers/self_monitoring/helpers.rb deleted file mode 100644 index 6956401e20d..00000000000 --- a/lib/gitlab/database_importers/self_monitoring/helpers.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module SelfMonitoring - module Helpers - def application_settings - @application_settings ||= ApplicationSetting.current_without_cache - end - - def project_created? - self_monitoring_project.present? - end - - def self_monitoring_project - application_settings.self_monitoring_project - end - - def self_monitoring_project_id - application_settings.self_monitoring_project_id - end - end - end - end -end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb deleted file mode 100644 index be500171bef..00000000000 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ /dev/null @@ -1,171 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module SelfMonitoring - module Project - class CreateService < ::BaseService - include Stepable - include SelfMonitoring::Helpers - - VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL - PROJECT_NAME = 'Monitoring' - - steps :validate_application_settings, - :create_group, - :create_project, - :save_project_id, - :create_environment, - :add_prometheus_manual_configuration, - :track_event - - def initialize - super(nil) - end - - def execute - execute_steps - end - - private - - def validate_application_settings(_result) - return success if application_settings - - log_error('No application_settings found') - error(_('No application_settings found')) - end - - def create_group(result) - create_group_response = - Gitlab::DatabaseImporters::InstanceAdministrators::CreateGroup.new.execute - - if create_group_response[:status] == :success - success(result.merge(create_group_response)) - else - error(create_group_response[:message]) - end - end - - def create_project(result) - if project_created? - log_info('Instance administration project already exists') - result[:project] = self_monitoring_project - return success(result) - end - - owner = result[:group].owners.first - - result[:project] = ::Projects::CreateService.new(owner, create_project_params(result[:group])).execute - - if result[:project].persisted? - success(result) - else - log_error("Could not create instance administration project. Errors: %{errors}" % { errors: result[:project].errors.full_messages }) - error(_('Could not create project')) - end - end - - def save_project_id(result) - return success(result) if project_created? - - response = application_settings.update( - self_monitoring_project_id: result[:project].id - ) - - if response - # In the add_prometheus_manual_configuration method, the Prometheus - # server_address config is saved as an api_url in the Integrations::Prometheus - # model. There are validates hooks in the Integrations::Prometheus model that - # check if the project associated with the Integrations::Prometheus is the - # self_monitoring project. It checks - # Gitlab::CurrentSettings.self_monitoring_project_id, which is why the - # Gitlab::CurrentSettings cache needs to be expired here, so that - # Integrations::Prometheus sees the latest self_monitoring_project_id. - Gitlab::CurrentSettings.expire_current_application_settings - success(result) - else - log_error("Could not save instance administration project ID, errors: %{errors}" % { errors: application_settings.errors.full_messages }) - error(_('Could not save project ID')) - end - end - - def create_environment(result) - return success(result) if result[:project].environments.exists? - - environment = ::Environment.new(project_id: result[:project].id, name: 'production') - - if environment.save - success(result) - else - log_error("Could not create environment for the Self-monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages }) - error(_('Could not create environment')) - end - end - - def add_prometheus_manual_configuration(result) - return success(result) unless prometheus_enabled? - return success(result) unless prometheus_server_address.present? - - prometheus = result[:project].find_or_initialize_integration('prometheus') - - unless prometheus.update(prometheus_integration_attributes) - log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: prometheus.errors.full_messages }) - return error(_('Could not save prometheus manual configuration')) - end - - success(result) - end - - def track_event(result) - project = result[:project] - ::Gitlab::Tracking.event("self_monitoring", "project_created", project: project, namespace: project.namespace) - - success(result) - end - - def parse_url(uri_string) - Addressable::URI.parse(uri_string) - rescue Addressable::URI::InvalidURIError, TypeError - end - - def prometheus_enabled? - ::Gitlab::Prometheus::Internal.prometheus_enabled? - end - - def prometheus_server_address - ::Gitlab::Prometheus::Internal.server_address - end - - def docs_path - Rails.application.routes.url_helpers.help_page_path( - 'administration/monitoring/gitlab_self_monitoring_project/index' - ) - end - - def create_project_params(group) - { - initialize_with_readme: true, - visibility_level: VISIBILITY_LEVEL, - name: PROJECT_NAME, - description: "This project is automatically generated and helps monitor this GitLab instance. [Learn more](#{docs_path}).", - namespace_id: group.id - } - end - - def internal_prometheus_server_address_uri - ::Gitlab::Prometheus::Internal.uri - end - - def prometheus_integration_attributes - { - api_url: internal_prometheus_server_address_uri, - manual_configuration: true, - active: true - } - end - end - end - end - end -end diff --git a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb deleted file mode 100644 index d5bed94d735..00000000000 --- a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module SelfMonitoring - module Project - class DeleteService < ::BaseService - include Stepable - include SelfMonitoring::Helpers - - steps :validate_self_monitoring_project_exists, - :destroy_project - - def initialize - super(nil) - end - - def execute - execute_steps - end - - private - - def validate_self_monitoring_project_exists(result) - unless project_created? || self_monitoring_project_id.present? - return error(_('Self-monitoring project does not exist')) - end - - success(result) - end - - def destroy_project(result) - return success(result) unless project_created? - - if self_monitoring_project.destroy - success(result) - else - log_error(self_monitoring_project.errors.full_messages) - error(_('Error deleting project. Check logs for error details.')) - end - end - end - end - end - end -end |