diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-09 18:08:16 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-09 18:08:16 +0000 |
| commit | 4da595a071829d1046f921e402f3978eeca15d93 (patch) | |
| tree | f792687f66b20dacfd5f48cae4ba42961bddddd7 | |
| parent | c87924a358cab4283e48f7c3ffd27b07320b9f62 (diff) | |
| download | gitlab-ce-4da595a071829d1046f921e402f3978eeca15d93.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
64 files changed, 1144 insertions, 437 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index fee2b6e6a15..f017abd9c4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -691,6 +691,16 @@ entry. - [Use `ssh_data` gem instead of `net-ssh` and `sshkey` where possible](gitlab-org/gitlab@59a0ee8605d509753c9aec719f8e0da77bcc679d) ([merge request](gitlab-org/gitlab!77424)) - [Remove feature flag already default enabled](gitlab-org/gitlab@9b7059a4bf9dc2ecdce1910a931cc6967d05b5ad) ([merge request](gitlab-org/gitlab!78238)) **GitLab Enterprise Edition** +## 14.7.5 (2022-03-09) + +### Fixed (1 change) + +- [Ensure cleanup job artifacts task does not include pipeline artifacts](gitlab-org/gitlab@7b5e91bc78c46109e48537b20239d4ab649a971a) ([merge request](gitlab-org/gitlab!82430)) + +### Other (1 change) + +- [Change to truncate table before adding finding_link_url_idx](gitlab-org/gitlab@6411ec61f40cb8648cea24ed26c1d69c8b910891) ([merge request](gitlab-org/gitlab!82430)) + ## 14.7.4 (2022-02-25) ### Security (8 changes) diff --git a/app/assets/javascripts/access_tokens/components/expires_at_field.vue b/app/assets/javascripts/access_tokens/components/expires_at_field.vue index 1fec186f2fa..561b2617c5f 100644 --- a/app/assets/javascripts/access_tokens/components/expires_at_field.vue +++ b/app/assets/javascripts/access_tokens/components/expires_at_field.vue @@ -1,15 +1,31 @@ <script> -import { GlDatepicker, GlFormInput } from '@gitlab/ui'; +import { GlDatepicker, GlFormInput, GlFormGroup } from '@gitlab/ui'; + +import { __ } from '~/locale'; export default { name: 'ExpiresAtField', - components: { GlDatepicker, GlFormInput }, + i18n: { + label: __('Expiration date'), + }, + components: { + GlDatepicker, + GlFormInput, + GlFormGroup, + MaxExpirationDateMessage: () => + import('ee_component/access_tokens/components/max_expiration_date_message.vue'), + }, props: { inputAttrs: { type: Object, required: false, default: () => ({}), }, + maxDate: { + type: Date, + required: false, + default: () => null, + }, }, data() { return { @@ -20,13 +36,18 @@ export default { </script> <template> - <gl-datepicker :target="null" :min-date="minDate"> - <gl-form-input - v-bind="inputAttrs" - class="datepicker gl-datepicker-input" - autocomplete="off" - inputmode="none" - data-qa-selector="expiry_date_field" - /> - </gl-datepicker> + <gl-form-group :label="$options.i18n.label" :label-for="inputAttrs.id"> + <gl-datepicker :target="null" :min-date="minDate" :max-date="maxDate"> + <gl-form-input + v-bind="inputAttrs" + class="datepicker gl-datepicker-input" + autocomplete="off" + inputmode="none" + data-qa-selector="expiry_date_field" + /> + </gl-datepicker> + <template #description> + <max-expiration-date-message :max-date="maxDate" /> + </template> + </gl-form-group> </template> diff --git a/app/assets/javascripts/access_tokens/index.js b/app/assets/javascripts/access_tokens/index.js index 9a1e7d877f8..c59bd445539 100644 --- a/app/assets/javascripts/access_tokens/index.js +++ b/app/assets/javascripts/access_tokens/index.js @@ -17,6 +17,7 @@ export const initExpiresAtField = () => { } const { expiresAt: inputAttrs } = parseRailsFormFields(el); + const { maxDate } = el.dataset; return new Vue({ el, @@ -24,6 +25,7 @@ export const initExpiresAtField = () => { return h(ExpiresAtField, { props: { inputAttrs, + maxDate: maxDate ? new Date(maxDate) : undefined, }, }); }, diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/details_header.vue b/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/details_header.vue index 29c181f04fb..ab0418388cd 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/details_header.vue +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/details_header.vue @@ -4,6 +4,7 @@ import { sprintf, n__, s__ } from '~/locale'; import MetadataItem from '~/vue_shared/components/registry/metadata_item.vue'; import TitleArea from '~/vue_shared/components/registry/title_area.vue'; import timeagoMixin from '~/vue_shared/mixins/timeago'; +import { numberToHumanSize } from '~/lib/utils/number_utils'; import { UPDATED_AT, CLEANUP_UNSCHEDULED_TEXT, @@ -23,7 +24,7 @@ import { ROOT_IMAGE_TOOLTIP, } from '../../constants/index'; -import getContainerRepositoryTagsCountQuery from '../../graphql/queries/get_container_repository_tags_count.query.graphql'; +import getContainerRepositoryMetadata from '../../graphql/queries/get_container_repository_metadata.query.graphql'; export default { name: 'DetailsHeader', @@ -50,7 +51,7 @@ export default { }, apollo: { containerRepository: { - query: getContainerRepositoryTagsCountQuery, + query: getContainerRepositoryMetadata, variables() { return { id: this.image.id, @@ -101,6 +102,10 @@ export default { imageName() { return this.imageDetails.name || ROOT_IMAGE_TEXT; }, + formattedSize() { + const { size } = this.imageDetails; + return size ? numberToHumanSize(Number(size)) : null; + }, }, }; </script> @@ -119,10 +124,15 @@ export default { :aria-label="rootImageTooltip" /> </template> + <template #metadata-tags-count> <metadata-item icon="tag" :text="tagCountText" data-testid="tags-count" /> </template> + <template v-if="formattedSize" #metadata-size> + <metadata-item icon="disk" :text="formattedSize" data-testid="image-size" /> + </template> + <template #metadata-cleanup> <metadata-item icon="expire" diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/graphql/queries/get_container_repository_tags_count.query.graphql b/app/assets/javascripts/packages_and_registries/container_registry/explorer/graphql/queries/get_container_repository_metadata.query.graphql index 9092a71edb0..f1f67b98407 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/graphql/queries/get_container_repository_tags_count.query.graphql +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/graphql/queries/get_container_repository_metadata.query.graphql @@ -1,6 +1,7 @@ -query getContainerRepositoryTagsCount($id: ID!) { +query getContainerRepositoryMetadata($id: ID!) { containerRepository(id: $id) { id tagsCount + size } } diff --git a/app/assets/javascripts/security_configuration/components/training_provider_list.vue b/app/assets/javascripts/security_configuration/components/training_provider_list.vue index 5bccd91e55e..4b40e5a1e1d 100644 --- a/app/assets/javascripts/security_configuration/components/training_provider_list.vue +++ b/app/assets/javascripts/security_configuration/components/training_provider_list.vue @@ -27,6 +27,17 @@ const i18n = { primaryTraining: s__('SecurityTraining|Primary Training'), }; +// Fetch the svg path from the GraphQL query once this issue is resolved +// https://gitlab.com/gitlab-org/gitlab/-/issues/346899 +const TEMP_PROVIDER_LOGOS = { + 'gid://gitlab/Security::TrainingProvider/1': { + svg: '/assets/illustrations/vulnerability/kontra-logo.svg', + }, + 'gid://gitlab/Security::TrainingProvider/2': { + svg: '/assets/illustrations/vulnerability/scw-logo.svg', + }, +}; + export default { components: { GlAlert, @@ -187,6 +198,7 @@ export default { }, }, i18n, + TEMP_PROVIDER_LOGOS, }; </script> @@ -215,7 +227,10 @@ export default { label-position="hidden" @change="toggleProvider(provider)" /> - <div class="gl-ml-5"> + <div v-if="$options.TEMP_PROVIDER_LOGOS[provider.id]" class="gl-ml-4"> + <img :src="$options.TEMP_PROVIDER_LOGOS[provider.id].svg" width="18" /> + </div> + <div class="gl-ml-3"> <h3 class="gl-font-lg gl-m-0 gl-mb-2">{{ provider.name }}</h3> <p> {{ provider.description }} diff --git a/app/assets/javascripts/vue_shared/components/filtered_search_bar/tokens/base_token.vue b/app/assets/javascripts/vue_shared/components/filtered_search_bar/tokens/base_token.vue index 157068b2c0f..389f2ca8ac2 100644 --- a/app/assets/javascripts/vue_shared/components/filtered_search_bar/tokens/base_token.vue +++ b/app/assets/javascripts/vue_shared/components/filtered_search_bar/tokens/base_token.vue @@ -78,7 +78,7 @@ export default { return { searchKey: '', recentSuggestions: this.config.recentSuggestionsStorageKey - ? getRecentlyUsedSuggestions(this.config.recentSuggestionsStorageKey) + ? getRecentlyUsedSuggestions(this.config.recentSuggestionsStorageKey) ?? [] : [], }; }, diff --git a/app/helpers/access_tokens_helper.rb b/app/helpers/access_tokens_helper.rb index 1d38262159f..d8d44601327 100644 --- a/app/helpers/access_tokens_helper.rb +++ b/app/helpers/access_tokens_helper.rb @@ -27,4 +27,10 @@ module AccessTokensHelper } }.to_json end + + def expires_at_field_data + {} + end end + +AccessTokensHelper.prepend_mod diff --git a/app/helpers/container_registry_helper.rb b/app/helpers/container_registry_helper.rb index 1b77b639ce1..255b8183164 100644 --- a/app/helpers/container_registry_helper.rb +++ b/app/helpers/container_registry_helper.rb @@ -2,8 +2,7 @@ module ContainerRegistryHelper def container_registry_expiration_policies_throttling? - Feature.enabled?(:container_registry_expiration_policies_throttling) && - ContainerRegistry::Client.supports_tag_delete? + Feature.enabled?(:container_registry_expiration_policies_throttling, default_enabled: :yaml) end def container_repository_gid_prefix diff --git a/app/models/concerns/integrations/has_issue_tracker_fields.rb b/app/models/concerns/integrations/has_issue_tracker_fields.rb new file mode 100644 index 00000000000..b1def38d019 --- /dev/null +++ b/app/models/concerns/integrations/has_issue_tracker_fields.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Integrations + module HasIssueTrackerFields + extend ActiveSupport::Concern + + included do + field :project_url, + required: true, + storage: :data_fields, + title: -> { _('Project URL') }, + help: -> { s_('IssueTracker|The URL to the project in the external issue tracker.') } + + field :issues_url, + required: true, + storage: :data_fields, + title: -> { s_('IssueTracker|Issue URL') }, + help: -> do + format s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.'), + colon_id: '<code>:id</code>'.html_safe + end + + field :new_issue_url, + required: true, + storage: :data_fields, + title: -> { s_('IssueTracker|New issue URL') }, + help: -> { s_('IssueTracker|The URL to create an issue in the external issue tracker.') } + end + end +end diff --git a/app/models/integration.rb b/app/models/integration.rb index fb44d90139a..3cf71f46420 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -122,6 +122,39 @@ class Integration < ApplicationRecord scope :alert_hooks, -> { where(alert_events: true, active: true) } scope :deployment, -> { where(category: 'deployment') } + class << self + private + + attr_writer :field_storage + + def field_storage + @field_storage || :properties + end + end + + # :nocov: Tested on subclasses. + def self.field(name, storage: field_storage, **attrs) + fields << ::Integrations::Field.new(name: name, **attrs) + + case storage + when :properties + prop_accessor(name) + when :data_fields + data_field(name) + else + raise ArgumentError, "Unknown field storage: #{storage}" + end + end + # :nocov: + + def self.fields + @fields ||= [] + end + + def fields + self.class.fields + end + # Provide convenient accessor methods for each serialized property. # Also keep track of updated properties in a similar way as ActiveModel::Dirty def self.prop_accessor(*args) @@ -395,11 +428,6 @@ class Integration < ApplicationRecord self.class.to_param end - def fields - # implement inside child - [] - end - def sections [] end diff --git a/app/models/integrations/base_issue_tracker.rb b/app/models/integrations/base_issue_tracker.rb index 71329f34dc1..458d0199e7a 100644 --- a/app/models/integrations/base_issue_tracker.rb +++ b/app/models/integrations/base_issue_tracker.rb @@ -4,10 +4,6 @@ module Integrations class BaseIssueTracker < Integration validate :one_issue_tracker, if: :activated?, on: :manual_change - # TODO: we can probably just delegate as part of - # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :project_url, :issues_url, :new_issue_url - default_value_for :category, 'issue_tracker' before_validation :handle_properties @@ -72,14 +68,6 @@ module Integrations issue_url(iid) end - def fields - [ - { type: 'text', name: 'project_url', title: _('Project URL'), help: s_('IssueTracker|The URL to the project in the external issue tracker.'), required: true }, - { type: 'text', name: 'issues_url', title: s_('IssueTracker|Issue URL'), help: s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.') % { colon_id: '<code>:id</code>'.html_safe }, required: true }, - { type: 'text', name: 'new_issue_url', title: s_('IssueTracker|New issue URL'), help: s_('IssueTracker|The URL to create an issue in the external issue tracker.'), required: true } - ] - end - def initialize_properties {} end diff --git a/app/models/integrations/bugzilla.rb b/app/models/integrations/bugzilla.rb index 1b711264e10..74e282f6848 100644 --- a/app/models/integrations/bugzilla.rb +++ b/app/models/integrations/bugzilla.rb @@ -2,6 +2,8 @@ module Integrations class Bugzilla < BaseIssueTracker + include Integrations::HasIssueTrackerFields + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? def title diff --git a/app/models/integrations/custom_issue_tracker.rb b/app/models/integrations/custom_issue_tracker.rb index 0299a50605f..3770e813eaa 100644 --- a/app/models/integrations/custom_issue_tracker.rb +++ b/app/models/integrations/custom_issue_tracker.rb @@ -2,6 +2,8 @@ module Integrations class CustomIssueTracker < BaseIssueTracker + include HasIssueTrackerFields + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? def title diff --git a/app/models/integrations/ewm.rb b/app/models/integrations/ewm.rb index c6ea989a03f..1b86ef73c85 100644 --- a/app/models/integrations/ewm.rb +++ b/app/models/integrations/ewm.rb @@ -2,6 +2,8 @@ module Integrations class Ewm < BaseIssueTracker + include HasIssueTrackerFields + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? def self.reference_pattern(only_long: true) diff --git a/app/models/integrations/field.rb b/app/models/integrations/field.rb new file mode 100644 index 00000000000..49ab97677db --- /dev/null +++ b/app/models/integrations/field.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Integrations + class Field + SENSITIVE_NAME = %r/token|key|password|passphrase|secret/.freeze + + ATTRIBUTES = %i[ + section type placeholder required choices value checkbox_label + title help + non_empty_password_help + non_empty_password_title + api_only + ].freeze + + attr_reader :name + + def initialize(name:, type: 'text', api_only: false, **attributes) + @name = name.to_s.freeze + + attributes[:type] = SENSITIVE_NAME.match?(@name) ? 'password' : type + attributes[:api_only] = api_only + @attributes = attributes.freeze + end + + def [](key) + return name if key == :name + + value = @attributes[key] + return value.call if value.respond_to?(:call) + + value + end + + def sensitive? + @attributes[:type] == 'password' + end + + ATTRIBUTES.each do |name| + define_method(name) { self[name] } + end + end +end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 8fc0a3461f7..7308c731bb0 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -28,11 +28,6 @@ module Integrations # We should use username/password for Jira Server and email/api_token for Jira Cloud, # for more information check: https://gitlab.com/gitlab-org/gitlab-foss/issues/49936. - # TODO: we can probably just delegate as part of - # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :username, :password, :url, :api_url, :jira_issue_transition_automatic, :jira_issue_transition_id, :project_key, :issues_enabled, - :vulnerabilities_enabled, :vulnerabilities_issuetype - before_validation :reset_password after_commit :update_deployment_type, on: [:create, :update], if: :update_deployment_type? @@ -41,6 +36,44 @@ module Integrations all_details: 2 } + self.field_storage = :data_fields + + field :url, + section: SECTION_TYPE_CONNECTION, + required: true, + title: -> { s_('JiraService|Web URL') }, + help: -> { s_('JiraService|Base URL of the Jira instance.') }, + placeholder: 'https://jira.example.com' + + field :api_url, + section: SECTION_TYPE_CONNECTION, + title: -> { s_('JiraService|Jira API URL') }, + help: -> { s_('JiraService|If different from Web URL.') } + + field :username, + section: SECTION_TYPE_CONNECTION, + required: true, + title: -> { s_('JiraService|Username or Email') }, + help: -> { s_('JiraService|Use a username for server version and an email for cloud version.') } + + field :password, + section: SECTION_TYPE_CONNECTION, + required: true, + title: -> { s_('JiraService|Password or API token') }, + non_empty_password_title: -> { s_('JiraService|Enter new password or API token') }, + non_empty_password_help: -> { s_('JiraService|Leave blank to use your current password or API token.') }, + help: -> { s_('JiraService|Use a password for server version and an API token for cloud version.') } + + # TODO: we can probably just delegate as part of + # https://gitlab.com/gitlab-org/gitlab/issues/29404 + # These fields are API only, so no field definition is required. + data_field :jira_issue_transition_automatic + data_field :jira_issue_transition_id + data_field :project_key + data_field :issues_enabled + data_field :vulnerabilities_enabled + data_field :vulnerabilities_issuetype + # When these are false GitLab does not create cross reference # comments on Jira except when an issue gets transitioned. def self.supported_events @@ -127,45 +160,6 @@ module Integrations 'jira' end - def fields - [ - { - section: SECTION_TYPE_CONNECTION, - type: 'text', - name: 'url', - title: s_('JiraService|Web URL'), - placeholder: 'https://jira.example.com', - help: s_('JiraService|Base URL of the Jira instance.'), - required: true - }, - { - section: SECTION_TYPE_CONNECTION, - type: 'text', - name: 'api_url', - title: s_('JiraService|Jira API URL'), - help: s_('JiraService|If different from Web URL.') - }, - { - section: SECTION_TYPE_CONNECTION, - type: 'text', - name: 'username', - title: s_('JiraService|Username or Email'), - help: s_('JiraService|Use a username for server version and an email for cloud version.'), - required: true - }, - { - section: SECTION_TYPE_CONNECTION, - type: 'password', - name: 'password', - title: s_('JiraService|Password or API token'), - non_empty_password_title: s_('JiraService|Enter new password or API token'), - non_empty_password_help: s_('JiraService|Leave blank to use your current password or API token.'), - help: s_('JiraService|Use a password for server version and an API token for cloud version.'), - required: true - } - ] - end - def sections [ { @@ -194,17 +188,12 @@ module Integrations url.to_s end - override :project_url - def project_url - web_url - end + alias_method :project_url, :web_url - override :issues_url def issues_url web_url('browse/:id') end - override :new_issue_url def new_issue_url web_url('secure/CreateIssue!default.jspa') end diff --git a/app/models/integrations/redmine.rb b/app/models/integrations/redmine.rb index 3812033c388..bc2a64b0848 100644 --- a/app/models/integrations/redmine.rb +++ b/app/models/integrations/redmine.rb @@ -2,6 +2,8 @@ module Integrations class Redmine < BaseIssueTracker + include Integrations::HasIssueTrackerFields + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? def title diff --git a/app/models/integrations/youtrack.rb b/app/models/integrations/youtrack.rb index 254f0c6c69a..ab6e1da27f8 100644 --- a/app/models/integrations/youtrack.rb +++ b/app/models/integrations/youtrack.rb @@ -2,6 +2,8 @@ module Integrations class Youtrack < BaseIssueTracker + include Integrations::HasIssueTrackerFields + validates :project_url, :issues_url, presence: true, public_url: true, if: :activated? # {PROJECT-KEY}-{NUMBER} Examples: YT-1, PRJ-1, gl-030 diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index e15ab1b037b..72f3fddb4c3 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -152,7 +152,7 @@ module Projects end def throttling_enabled? - Feature.enabled?(:container_registry_expiration_policies_throttling) + Feature.enabled?(:container_registry_expiration_policies_throttling, default_enabled: :yaml) end def max_list_size diff --git a/app/services/projects/container_repository/gitlab/delete_tags_service.rb b/app/services/projects/container_repository/gitlab/delete_tags_service.rb index 589aac5c3ac..f109cb0ca20 100644 --- a/app/services/projects/container_repository/gitlab/delete_tags_service.rb +++ b/app/services/projects/container_repository/gitlab/delete_tags_service.rb @@ -54,7 +54,7 @@ module Projects def throttling_enabled? strong_memoize(:feature_flag) do - Feature.enabled?(:container_registry_expiration_policies_throttling) + Feature.enabled?(:container_registry_expiration_policies_throttling, default_enabled: :yaml) end end diff --git a/app/views/groups/_shared_projects.html.haml b/app/views/groups/_shared_projects.html.haml index bfd056ccdd2..ef6410ad439 100644 --- a/app/views/groups/_shared_projects.html.haml +++ b/app/views/groups/_shared_projects.html.haml @@ -4,5 +4,4 @@ %ul.content-list{ data: { hide_projects: 'false', group_id: group.id, path: group_path(group) } } .js-groups-list-holder - .loading-container.text-center.prepend-top-20 - .gl-spinner.gl-spinner-md + = gl_loading_icon diff --git a/app/views/shared/access_tokens/_form.html.haml b/app/views/shared/access_tokens/_form.html.haml index 7e6e8e06397..0b68cfe65e5 100644 --- a/app/views/shared/access_tokens/_form.html.haml +++ b/app/views/shared/access_tokens/_form.html.haml @@ -24,14 +24,9 @@ %span.form-text.text-muted.col-md-12#access_token_help_text= _("For example, the application using the token or the purpose of the token. Do not give sensitive information for the name of the token, as it will be visible to all %{resource_type} members.") % { resource_type: resource_type } .row - .form-group.col-md-6 - = f.label :expires_at, _('Expiration date'), class: 'label-bold' - .input-icon-wrapper - - = render_if_exists 'personal_access_tokens/callout_max_personal_access_token_lifetime' - - .js-access-tokens-expires-at - = f.text_field :expires_at, class: 'datepicker gl-datepicker-input form-control gl-form-input', placeholder: 'YYYY-MM-DD', autocomplete: 'off', data: { js_name: 'expiresAt' } + .col + .js-access-tokens-expires-at{ data: expires_at_field_data } + = f.text_field :expires_at, class: 'datepicker gl-datepicker-input form-control gl-form-input', placeholder: 'YYYY-MM-DD', autocomplete: 'off', data: { js_name: 'expiresAt' } - if resource .row diff --git a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb index 7f7a77d0524..cd3ed5d4c9b 100644 --- a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb +++ b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb @@ -123,7 +123,7 @@ module ContainerExpirationPolicies end def throttling_enabled? - Feature.enabled?(:container_registry_expiration_policies_throttling) + Feature.enabled?(:container_registry_expiration_policies_throttling, default_enabled: :yaml) end def max_cleanup_execution_time diff --git a/app/workers/container_expiration_policy_worker.rb b/app/workers/container_expiration_policy_worker.rb index 16ac61976eb..308ccfe2cb3 100644 --- a/app/workers/container_expiration_policy_worker.rb +++ b/app/workers/container_expiration_policy_worker.rb @@ -99,7 +99,7 @@ class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWo end def throttling_enabled? - Feature.enabled?(:container_registry_expiration_policies_throttling) + Feature.enabled?(:container_registry_expiration_policies_throttling, default_enabled: :yaml) end def lease_timeout diff --git a/config.ru b/config.ru index c74a49cd0e2..f07c25f503a 100644 --- a/config.ru +++ b/config.ru @@ -4,17 +4,7 @@ require ::File.expand_path('../config/environment', __FILE__) -def master_process? - Prometheus::PidProvider.worker_id == 'puma_master' -end - warmup do |app| - # The following is necessary to ensure stale Prometheus metrics don't accumulate over time. - # It needs to be done as early as here to ensure metrics files aren't deleted. - # After we hit our app in `warmup`, first metrics and corresponding files already being created, - # for example in `lib/gitlab/metrics/requests_rack_middleware.rb`. - Prometheus::CleanupMultiprocDirService.new.execute if master_process? - client = Rack::MockRequest.new(app) client.get('/') end diff --git a/config/feature_flags/development/container_registry_expiration_policies_throttling.yml b/config/feature_flags/development/container_registry_expiration_policies_throttling.yml index f9aa6bde700..e22c7c3177b 100644 --- a/config/feature_flags/development/container_registry_expiration_policies_throttling.yml +++ b/config/feature_flags/development/container_registry_expiration_policies_throttling.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/238190 milestone: '13.4' type: development group: group::package -default_enabled: false +default_enabled: true diff --git a/config/feature_flags/development/preserve_latest_wal_locations_for_idempotent_jobs.yml b/config/feature_flags/development/spread_parallel_import.yml index 24e4823997d..e47a4c1e676 100644 --- a/config/feature_flags/development/preserve_latest_wal_locations_for_idempotent_jobs.yml +++ b/config/feature_flags/development/spread_parallel_import.yml @@ -1,8 +1,8 @@ --- -name: preserve_latest_wal_locations_for_idempotent_jobs -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66280 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/338350 -milestone: '14.3' +name: spread_parallel_import +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81026 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/353217 +milestone: '14.9' type: development -group: group::memory -default_enabled: true +group: group::source code +default_enabled: false diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 9e9d02c24c5..a65b1041d83 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -13,6 +13,24 @@ def prometheus_default_multiproc_dir end end +def puma_metrics_server_process? + Prometheus::PidProvider.worker_id == 'puma_master' +end + +def sidekiq_metrics_server_process? + Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER_ID'] == '0') +end + +if puma_metrics_server_process? || sidekiq_metrics_server_process? + # The following is necessary to ensure stale Prometheus metrics don't accumulate over time. + # It needs to be done as early as here to ensure metrics files aren't deleted. + # After we hit our app in `warmup`, first metrics and corresponding files already being created, + # for example in `lib/gitlab/metrics/requests_rack_middleware.rb`. + Prometheus::CleanupMultiprocDirService.new.execute + + ::Prometheus::Client.reinitialize_on_pid_change(force: true) +end + ::Prometheus::Client.configure do |config| config.logger = Gitlab::AppLogger @@ -49,15 +67,9 @@ if Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER end Gitlab::Cluster::LifecycleEvents.on_master_start do - # When running Puma in a Single mode, `on_master_start` and `on_worker_start` are the same. - # Thus, we order these events to run `reinitialize_on_pid_change` with `force: true` first. - ::Prometheus::Client.reinitialize_on_pid_change(force: true) - Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1) if Gitlab::Runtime.puma? - Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics - Gitlab::Metrics::Samplers::PumaSampler.instance.start # Starts a metrics server to export metrics from the Puma primary. diff --git a/config/initializers/zz_metrics.rb b/config/initializers/zz_metrics.rb index 7fa71225aae..88469d2cdef 100644 --- a/config/initializers/zz_metrics.rb +++ b/config/initializers/zz_metrics.rb @@ -36,6 +36,10 @@ if Gitlab::Metrics.enabled? && !Rails.env.test? && !(Rails.env.development? && d config.middleware.use(Gitlab::Metrics::ElasticsearchRackMiddleware) end + if Gitlab::Runtime.puma? + Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics + end + GC::Profiler.enable module TrackNewRedisConnections diff --git a/config/metrics/counts_28d/20210216181937_failed_deployments.yml b/config/metrics/counts_28d/20210216181937_failed_deployments.yml index 9ef4157ce2d..78622151915 100644 --- a/config/metrics/counts_28d/20210216181937_failed_deployments.yml +++ b/config/metrics/counts_28d/20210216181937_failed_deployments.yml @@ -1,7 +1,7 @@ --- data_category: optional key_path: usage_activity_by_stage_monthly.release.failed_deployments -description: Total failed deployments +description: Disinct users who initiated a failed deployment. product_section: ops product_stage: release product_group: group::release diff --git a/config/metrics/counts_28d/20210216181941_successful_deployments.yml b/config/metrics/counts_28d/20210216181941_successful_deployments.yml index f21cb609208..10c9de1817f 100644 --- a/config/metrics/counts_28d/20210216181941_successful_deployments.yml +++ b/config/metrics/counts_28d/20210216181941_successful_deployments.yml @@ -1,7 +1,7 @@ --- data_category: optional key_path: usage_activity_by_stage_monthly.release.successful_deployments -description: Total successful deployments +description: Disinct users who initiated a successful deployment. product_section: ops product_stage: release product_group: group::release diff --git a/db/post_migrate/20220304165107_drop_partitioned_foreign_keys.rb b/db/post_migrate/20220304165107_drop_partitioned_foreign_keys.rb new file mode 100644 index 00000000000..43f89b05fa4 --- /dev/null +++ b/db/post_migrate/20220304165107_drop_partitioned_foreign_keys.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class DropPartitionedForeignKeys < Gitlab::Database::Migration[1.0] + def up + drop_table :partitioned_foreign_keys + end + + def down + create_table :partitioned_foreign_keys do |t| + t.boolean :cascade_delete, null: false, default: true + t.text :from_table, null: false, limit: 63 + t.text :from_column, null: false, limit: 63 + t.text :to_table, null: false, limit: 63 + t.text :to_column, null: false, limit: 63 + + t.index [:to_table, :from_table, :from_column], unique: true, name: :index_partitioned_foreign_keys_unique_index + end + end +end diff --git a/db/schema_migrations/20220304165107 b/db/schema_migrations/20220304165107 new file mode 100644 index 00000000000..6db7aee6b0f --- /dev/null +++ b/db/schema_migrations/20220304165107 @@ -0,0 +1 @@ +b7090327d2638bbee6646e5ca5a8f8597d97631f10f997698b8a1c1b6329c106
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 0a4b3bec84a..601005185a3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -18497,28 +18497,6 @@ CREATE SEQUENCE pages_domains_id_seq ALTER SEQUENCE pages_domains_id_seq OWNED BY pages_domains.id; -CREATE TABLE partitioned_foreign_keys ( - id bigint NOT NULL, - cascade_delete boolean DEFAULT true NOT NULL, - from_table text NOT NULL, - from_column text NOT NULL, - to_table text NOT NULL, - to_column text NOT NULL, - CONSTRAINT check_2c2e02a62b CHECK ((char_length(from_column) <= 63)), - CONSTRAINT check_40738efb57 CHECK ((char_length(to_table) <= 63)), - CONSTRAINT check_741676d405 CHECK ((char_length(from_table) <= 63)), - CONSTRAINT check_7e98be694f CHECK ((char_length(to_column) <= 63)) -); - -CREATE SEQUENCE partitioned_foreign_keys_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE partitioned_foreign_keys_id_seq OWNED BY partitioned_foreign_keys.id; - CREATE TABLE path_locks ( id integer NOT NULL, path character varying NOT NULL, @@ -22862,8 +22840,6 @@ ALTER TABLE ONLY pages_domain_acme_orders ALTER COLUMN id SET DEFAULT nextval('p ALTER TABLE ONLY pages_domains ALTER COLUMN id SET DEFAULT nextval('pages_domains_id_seq'::regclass); -ALTER TABLE ONLY partitioned_foreign_keys ALTER COLUMN id SET DEFAULT nextval('partitioned_foreign_keys_id_seq'::regclass); - ALTER TABLE ONLY path_locks ALTER COLUMN id SET DEFAULT nextval('path_locks_id_seq'::regclass); ALTER TABLE ONLY personal_access_tokens ALTER COLUMN id SET DEFAULT nextval('personal_access_tokens_id_seq'::regclass); @@ -24916,9 +24892,6 @@ ALTER TABLE ONLY pages_domain_acme_orders ALTER TABLE ONLY pages_domains ADD CONSTRAINT pages_domains_pkey PRIMARY KEY (id); -ALTER TABLE ONLY partitioned_foreign_keys - ADD CONSTRAINT partitioned_foreign_keys_pkey PRIMARY KEY (id); - ALTER TABLE ONLY path_locks ADD CONSTRAINT path_locks_pkey PRIMARY KEY (id); @@ -28509,8 +28482,6 @@ CREATE UNIQUE INDEX index_partial_am_alerts_on_project_id_and_fingerprint ON ale CREATE INDEX index_partial_ci_builds_on_user_id_name_parser_features ON ci_builds USING btree (user_id, name) WHERE (((type)::text = 'Ci::Build'::text) AND ((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('license_scanning'::character varying)::text, ('sast'::character varying)::text, ('coverage_fuzzing'::character varying)::text, ('secret_detection'::character varying)::text]))); -CREATE UNIQUE INDEX index_partitioned_foreign_keys_unique_index ON partitioned_foreign_keys USING btree (to_table, from_table, from_column); - CREATE INDEX index_pat_on_user_id_and_expires_at ON personal_access_tokens USING btree (user_id, expires_at); CREATE INDEX index_path_locks_on_path ON path_locks USING btree (path); diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index b1d0b6c2351..8677d5b08e3 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -7,9 +7,10 @@ type: index, concepts, howto # CI/CD development documentation **(FREE)** -Development guides that are specific to CI/CD are listed here. +Development guides that are specific to CI/CD are listed here: -If you are creating new CI/CD templates, please read [the development guide for GitLab CI/CD templates](templates.md). +- If you are creating new CI/CD templates, please read [the development guide for GitLab CI/CD templates](templates.md). +- If you are adding a new keyword or changing the CI schema, check the [CI schema guide](schema.md) See the [CI/CD YAML reference documentation guide](cicd_reference_documentation_guide.md) to learn how to update the [reference page](../../ci/yaml/index.md). diff --git a/doc/development/cicd/schema.md b/doc/development/cicd/schema.md new file mode 100644 index 00000000000..b63d951b881 --- /dev/null +++ b/doc/development/cicd/schema.md @@ -0,0 +1,146 @@ +--- +stage: Verify +group: Pipeline Authoring +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: index, howto +--- + +# Contribute to the CI Schema **(FREE)** + +The [pipeline editor](../../ci/pipeline_editor/index.md) uses a CI schema to enhance +the authoring experience of our CI configuration files. With the CI schema, the editor can: + +- Validate the content of the CI configuration file as it is being written in the editor. +- Provide autocomplete functionality and suggest available keywords. +- Provide definitions of keywords through annotations. + +As the rules and keywords for configuring our CI configuration files change, so too +should our CI schema. + +This feature is behind the [`schema_linting`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/feature_flags/development/schema_linting.yml) +feature flag for self-managed instances, and is enabled for GitLab.com. + +## JSON Schemas + +The CI schema follows the [JSON Schema Draft-07](https://json-schema.org/draft-07/json-schema-release-notes.html) +specification. Although the CI configuration file is written in YAML, it is converted +into JSON by using `monaco-yaml` before it is validated by the CI schema. + +If you're new to JSON schemas, consider checking out +[this guide](https://json-schema.org/learn/getting-started-step-by-step) for +a step-by-step introduction on how to work with JSON schemas. + +## Update Keywords + +The CI schema is at [`app/assets/javascripts/editor/schema/ci.json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json). +It contains all the keywords available for authoring CI configuration files. +Check the [keyword reference](../../ci/yaml/index.md) for a comprehensive list of +all available keywords. + +All keywords are defined under `definitions`. We use these definitions as +[references](https://json-schema.org/learn/getting-started-step-by-step#references) +to share common data structures across the schema. + +For example, this defines the `retry` keyword: + +```json +{ + "definitions": { + "retry": { + "description": "Retry a job if it fails. Can be a simple integer or object definition.", + "oneOf": [ + { + "$ref": "#/definitions/retry_max" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "max": { + "$ref": "#/definitions/retry_max" + }, + "when": { + "description": "Either a single or array of error types to trigger job retry.", + "oneOf": [ + { + "$ref": "#/definitions/retry_errors" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/retry_errors" + } + } + ] + } + } + } + ] + } + } +} +``` + +With this definition, the `retry` keyword is both a property of +the `job_template` definition and the `default` global keyword. Global keywords +that configure pipeline behavior (such as `workflow` and `stages`) are defined +under the topmost **properties** key. + +```json +{ + "properties": { + "default": { + "type": "object", + "properties": { + "retry": { + "$ref": "#/definitions/retry" + }, + } + } + }, + "definitions": { + "job_template": { + "properties": { + "retry": { + "$ref": "#/definitions/retry" + } + }, + } + } +} +``` + +## Guidelines for updating the schema + +- Keep definitions atomic when possible, to be flexible with + referencing keywords. For example, `workflow:rules` uses only a subset of + properties in the `rules` definition. The `rules` properties have their + own definitions, so we can reference them individually. +- When adding new keywords, consider adding a `description` with a link to the + keyword definition in the documentation. This information shows up in the annotations + when the user hovers over the keyword. +- For each property, consider if a `minimum`, `maximum`, or + `default` values are required. Some values might be required, and in others we can set + blank. In the blank case, we can add the following to the definition: + +```json +{ + "keyword": { + "oneOf": [ + { + "type": "null" + }, + ... + ] + } +} +``` + +## Test the schema + +For now, the CI schema can only be tested manually. To verify the behavior is correct: + +1. Enable the `schema_linting` feature flag. +1. Go to **CI/CD** > **Editor**. +1. Write your CI/CD configuration in the editor and verify that the schema validates + it correctly. diff --git a/doc/development/sidekiq/idempotent_jobs.md b/doc/development/sidekiq/idempotent_jobs.md index cea550d892e..38db22f8467 100644 --- a/doc/development/sidekiq/idempotent_jobs.md +++ b/doc/development/sidekiq/idempotent_jobs.md @@ -190,6 +190,7 @@ that can tolerate some duplication. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69372) in GitLab 14.3. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/338350) in GitLab 14.4. > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338350) in GitLab 14.6. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/346598) in GitLab 14.9. [Feature flag preserve_latest_wal_locations_for_idempotent_jobs](https://gitlab.com/gitlab-org/gitlab/-/issues/346598) removed. The deduplication always take into account the latest binary replication pointer, not the first one. This happens because we drop the same job scheduled for the second time and the Write-Ahead Log (WAL) is lost. @@ -199,10 +200,3 @@ To support both deduplication and maintaining data consistency with load balanci we are preserving the latest WAL location for idempotent jobs in Redis. This way we are always comparing the latest binary replication pointer, making sure that we read from the replica that is fully caught up. - -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to -[disable the feature flag](../../administration/feature_flags.md) named `preserve_latest_wal_locations_for_idempotent_jobs`. - -This feature flag is related to GitLab development and is not intended to be used by GitLab administrators, though. -On GitLab.com, this feature is available. diff --git a/doc/operations/incident_management/img/incident_metrics_tab_text_link_modal_v14_9.png b/doc/operations/incident_management/img/incident_metrics_tab_text_link_modal_v14_9.png Binary files differnew file mode 100644 index 00000000000..1b045a13fc5 --- /dev/null +++ b/doc/operations/incident_management/img/incident_metrics_tab_text_link_modal_v14_9.png diff --git a/doc/operations/incident_management/img/metric_image_url_dialog_v13_8.png b/doc/operations/incident_management/img/metric_image_url_dialog_v13_8.png Binary files differdeleted file mode 100644 index 732921bbb9f..00000000000 --- a/doc/operations/incident_management/img/metric_image_url_dialog_v13_8.png +++ /dev/null diff --git a/doc/operations/incident_management/incidents.md b/doc/operations/incident_management/incidents.md index bbaec853461..cc33fead4e3 100644 --- a/doc/operations/incident_management/incidents.md +++ b/doc/operations/incident_management/incidents.md @@ -173,9 +173,11 @@ charts in the **Metrics** tab:  -When you upload an image, you can associate it with a URL to the original graph. Users can access the original graph by clicking the image: +When you upload an image, you can associate the image with text or a link to the original graph. - + + +If you add a link, you can access the original graph by clicking the hyperlink above the uploaded image. ### Alert details diff --git a/doc/user/packages/container_registry/reduce_container_registry_storage.md b/doc/user/packages/container_registry/reduce_container_registry_storage.md index 2c96bf5f161..7e8b2865b6e 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_storage.md +++ b/doc/user/packages/container_registry/reduce_container_registry_storage.md @@ -157,11 +157,13 @@ Here are examples of regex patterns you may want to use: ### Set cleanup limits to conserve resources -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/288812) in GitLab 13.9. -> - It's [deployed behind a feature flag](../../feature_flags.md), disabled by default. -> - It's enabled on GitLab.com. -> - It's not recommended for production use. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-cleanup-policy-limits). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/288812) in GitLab 13.9 [with a flag](../../../administration/feature_flags.md) named `container_registry_expiration_policies_throttling`. Disabled by default. +> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80815) in GitLab 14.9. + +FLAG: +By default this feature is available in GitLab 14.9. To disable the feature, an administrator can +[disable the feature flag](../../../administration/feature_flags.md) +named `container_registry_expiration_policies_throttling`. Cleanup policies are executed as a background process. This process is complex, and depending on the number of tags to delete, the process can take time to finish. @@ -188,31 +190,11 @@ For self-managed instances, those settings can be updated in the [Rails console] ApplicationSetting.last.update(container_registry_expiration_policies_worker_capacity: 3) ``` -Alternatively, once the limits are [enabled](#enable-or-disable-cleanup-policy-limits), -they are available in the [administrator area](../../admin_area/index.md): +They are also available in the [administrator area](../../admin_area/index.md): 1. On the top bar, select **Menu > Admin**. 1. Go to **Settings > CI/CD > Container Registry**. -#### Enable or disable cleanup policy limits - -The cleanup policies limits are under development and not ready for production use. They are -deployed behind a feature flag that is **disabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can enable it. - -To enable it: - -```ruby -Feature.enable(:container_registry_expiration_policies_throttling) -``` - -To disable it: - -```ruby -Feature.disable(:container_registry_expiration_policies_throttling) -``` - ### Use the cleanup policy API You can set, update, and disable the cleanup policies using the GitLab API. diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 6159fb0a811..e423d1f17da 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -70,7 +70,7 @@ module Gitlab # Hook registration methods (called from initializers) # def on_worker_start(&block) - if in_clustered_environment? + if in_clustered_puma? # Defer block execution (@worker_start_hooks ||= []) << block else @@ -101,7 +101,7 @@ module Gitlab end def on_master_start(&block) - if in_clustered_environment? + if in_clustered_puma? on_before_fork(&block) else on_worker_start(&block) @@ -158,21 +158,8 @@ module Gitlab end end - def in_clustered_environment? - # Sidekiq doesn't fork - return false if Gitlab::Runtime.sidekiq? - - # Puma sometimes forks - return true if in_clustered_puma? - - # Default assumption is that we don't fork - false - end - def in_clustered_puma? - return false unless Gitlab::Runtime.puma? - - @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0 + Gitlab::Runtime.puma? && @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0 end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 4c7e0a4e613..436d5131d98 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -381,7 +381,6 @@ pages_deployments: :gitlab_main pages_deployment_states: :gitlab_main pages_domain_acme_orders: :gitlab_main pages_domains: :gitlab_main -partitioned_foreign_keys: :gitlab_main path_locks: :gitlab_main personal_access_tokens: :gitlab_main plan_limits: :gitlab_main diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index fc0c099b71c..5d291d9d723 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -74,6 +74,10 @@ module Gitlab { state: 'all', sort: 'created', direction: 'asc' } end + def parallel_import_batch + { size: 200, delay: 1.minute } + end + def repository_updates_counter @repository_updates_counter ||= Gitlab::Metrics.counter( :github_importer_repository_updates, diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index a8e006ea082..4dec9543a13 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -72,6 +72,14 @@ module Gitlab # Imports all objects in parallel by scheduling a Sidekiq job for every # individual object. def parallel_import + if Feature.enabled?(:spread_parallel_import, default_enabled: :yaml) && parallel_import_batch.present? + spread_parallel_import + else + parallel_import_deprecated + end + end + + def parallel_import_deprecated waiter = JobWaiter.new each_object_to_import do |object| @@ -86,6 +94,33 @@ module Gitlab waiter end + def spread_parallel_import + waiter = JobWaiter.new + + import_arguments = [] + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + import_arguments << [project.id, repr.to_hash, waiter.key] + + waiter.jobs_remaining += 1 + end + + # rubocop:disable Scalability/BulkPerformWithContext + Gitlab::ApplicationContext.with_context(project: project) do + sidekiq_worker_class.bulk_perform_in( + 1.second, + import_arguments, + batch_size: parallel_import_batch[:size], + batch_delay: parallel_import_batch[:delay] + ) + end + # rubocop:enable Scalability/BulkPerformWithContext + + waiter + end + # The method that will be called for traversing through all the objects to # import, yielding them to the supplied block. def each_object_to_import @@ -171,6 +206,12 @@ module Gitlab raise NotImplementedError end + # Default batch settings for parallel import (can be redefined in Importer classes) + # Example: { size: 100, delay: 1.minute } + def parallel_import_batch + {} + end + def abort_on_failure false end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 5671fce481f..e2df60c46f1 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -62,10 +62,6 @@ module Gitlab end def self.quiet - # Disable database insertion logs so speed isn't limited by ability to print to console - old_logger = ActiveRecord::Base.logger - ActiveRecord::Base.logger = nil - # Additional seed logic for models. Project.include(ProjectSeed) User.include(UserSeed) @@ -75,9 +71,11 @@ module Gitlab SeedFu.quiet = true - without_statement_timeout do - without_new_note_notifications do - yield + without_database_logging do + without_statement_timeout do + without_new_note_notifications do + yield + end end end @@ -85,7 +83,6 @@ module Gitlab ensure SeedFu.quiet = false ActionMailer::Base.perform_deliveries = old_perform_deliveries - ActiveRecord::Base.logger = old_logger end def self.without_gitaly_timeout @@ -112,10 +109,30 @@ module Gitlab end def self.without_statement_timeout - ActiveRecord::Base.connection.execute('SET statement_timeout=0') + Gitlab::Database::EachDatabase.each_database_connection do |connection| + connection.execute('SET statement_timeout=0') + end + yield + ensure + Gitlab::Database::EachDatabase.each_database_connection do |connection| + connection.execute('RESET statement_timeout') + end + end + + def self.without_database_logging + old_loggers = Gitlab::Database.database_base_models.transform_values do |model| + model.logger + end + + Gitlab::Database.database_base_models.each do |_, model| + model.logger = nil + end + yield ensure - ActiveRecord::Base.connection.execute('RESET statement_timeout') + Gitlab::Database.database_base_models.each do |connection_name, model| + model.logger = old_loggers[connection_name] + end end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index f31262bfcc9..601c8d1c3cf 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -167,7 +167,6 @@ module Gitlab def idempotent? return false unless worker_klass return false unless worker_klass.respond_to?(:idempotent?) - return false unless preserve_wal_location? || !worker_klass.utilizes_load_balancing_capabilities? worker_klass.idempotent? end @@ -206,8 +205,6 @@ module Gitlab end def job_wal_locations - return {} unless preserve_wal_location? - job['wal_locations'] || {} end @@ -272,10 +269,6 @@ module Gitlab @existing_wal_locations ||= {} end - def preserve_wal_location? - Feature.enabled?(:preserve_latest_wal_locations_for_idempotent_jobs, default_enabled: :yaml) - end - def reschedulable? !scheduled? && options[:if_deduplicated] == :reschedule_once end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 5762797d622..dd0310fd630 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -3702,6 +3702,9 @@ msgstr "" msgid "An %{link_start}alert%{link_end} with the same fingerprint is already open. To change the status of this alert, resolve the linked alert." msgstr "" +msgid "An Administrator has set the maximum expiration date to %{maxDate}. %{helpLinkStart}Learn more%{helpLinkEnd}." +msgstr "" + msgid "An Enterprise User GitLab account has been created for you by your organization:" msgstr "" @@ -22659,9 +22662,6 @@ msgstr "" msgid "Maximum job timeout has a value which could not be accepted" msgstr "" -msgid "Maximum lifetime allowable for Personal Access Tokens is active, your expire date must be set before %{maximum_allowable_date}." -msgstr "" - msgid "Maximum lines in a diff" msgstr "" diff --git a/spec/features/admin/admin_settings_spec.rb b/spec/features/admin/admin_settings_spec.rb index b6dcc5df7be..33e22c377a3 100644 --- a/spec/features/admin/admin_settings_spec.rb +++ b/spec/features/admin/admin_settings_spec.rb @@ -394,26 +394,16 @@ RSpec.describe 'Admin updates settings' do %i[container_registry_delete_tags_service_timeout container_registry_expiration_policies_worker_capacity container_registry_cleanup_tags_service_max_list_size].each do |setting| context "for container registry setting #{setting}" do - context 'with feature flag enabled' do - context 'with client supporting tag delete' do - it 'changes the setting' do - visit ci_cd_admin_application_settings_path - - page.within('.as-registry') do - fill_in "application_setting_#{setting}", with: 400 - click_button 'Save changes' - end - - expect(current_settings.public_send(setting)).to eq(400) - expect(page).to have_content "Application settings saved successfully" - end - end - - context 'with client not supporting tag delete' do - let(:client_support) { false } + it 'changes the setting' do + visit ci_cd_admin_application_settings_path - it_behaves_like 'not having container registry setting', setting + page.within('.as-registry') do + fill_in "application_setting_#{setting}", with: 400 + click_button 'Save changes' end + + expect(current_settings.public_send(setting)).to eq(400) + expect(page).to have_content "Application settings saved successfully" end context 'with feature flag disabled' do @@ -425,28 +415,18 @@ RSpec.describe 'Admin updates settings' do end context 'for container registry setting container_registry_expiration_policies_caching' do - context 'with feature flag enabled' do - context 'with client supporting tag delete' do - it 'updates container_registry_expiration_policies_caching' do - old_value = current_settings.container_registry_expiration_policies_caching + it 'updates container_registry_expiration_policies_caching' do + old_value = current_settings.container_registry_expiration_policies_caching - visit ci_cd_admin_application_settings_path - - page.within('.as-registry') do - find('#application_setting_container_registry_expiration_policies_caching.form-check-input').click - click_button 'Save changes' - end + visit ci_cd_admin_application_settings_path - expect(current_settings.container_registry_expiration_policies_caching).to eq(!old_value) - expect(page).to have_content "Application settings saved successfully" - end + page.within('.as-registry') do + find('#application_setting_container_registry_expiration_policies_caching.form-check-input').click + click_button 'Save changes' end - context 'with client not supporting tag delete' do - let(:client_support) { false } - - it_behaves_like 'not having container registry setting', :container_registry_expiration_policies_caching - end + expect(current_settings.container_registry_expiration_policies_caching).to eq(!old_value) + expect(page).to have_content "Application settings saved successfully" end context 'with feature flag disabled' do diff --git a/spec/frontend/access_tokens/components/__snapshots__/expires_at_field_spec.js.snap b/spec/frontend/access_tokens/components/__snapshots__/expires_at_field_spec.js.snap index 0b86c10ea46..dd742419d32 100644 --- a/spec/frontend/access_tokens/components/__snapshots__/expires_at_field_spec.js.snap +++ b/spec/frontend/access_tokens/components/__snapshots__/expires_at_field_spec.js.snap @@ -1,25 +1,32 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`~/access_tokens/components/expires_at_field should render datepicker with input info 1`] = ` -<gl-datepicker-stub - ariallabel="" - autocomplete="" - container="" - displayfield="true" - firstday="0" - inputlabel="Enter date" - mindate="Mon Jul 06 2020 00:00:00 GMT+0000 (Greenwich Mean Time)" - placeholder="YYYY-MM-DD" - theme="" +<gl-form-group-stub + label="Expiration date" + label-for="personal_access_token_expires_at" + labeldescription="" + optionaltext="(optional)" > - <gl-form-input-stub - autocomplete="off" - class="datepicker gl-datepicker-input" - data-qa-selector="expiry_date_field" - id="personal_access_token_expires_at" - inputmode="none" - name="personal_access_token[expires_at]" + <gl-datepicker-stub + ariallabel="" + autocomplete="" + container="" + displayfield="true" + firstday="0" + inputlabel="Enter date" + mindate="Mon Jul 06 2020 00:00:00 GMT+0000 (Greenwich Mean Time)" placeholder="YYYY-MM-DD" - /> -</gl-datepicker-stub> + theme="" + > + <gl-form-input-stub + autocomplete="off" + class="datepicker gl-datepicker-input" + data-qa-selector="expiry_date_field" + id="personal_access_token_expires_at" + inputmode="none" + name="personal_access_token[expires_at]" + placeholder="YYYY-MM-DD" + /> + </gl-datepicker-stub> +</gl-form-group-stub> `; diff --git a/spec/frontend/access_tokens/components/expires_at_field_spec.js b/spec/frontend/access_tokens/components/expires_at_field_spec.js index 4a2815e6931..fc8edcb573f 100644 --- a/spec/frontend/access_tokens/components/expires_at_field_spec.js +++ b/spec/frontend/access_tokens/components/expires_at_field_spec.js @@ -4,15 +4,17 @@ import ExpiresAtField from '~/access_tokens/components/expires_at_field.vue'; describe('~/access_tokens/components/expires_at_field', () => { let wrapper; - const createComponent = () => { + const defaultPropsData = { + inputAttrs: { + id: 'personal_access_token_expires_at', + name: 'personal_access_token[expires_at]', + placeholder: 'YYYY-MM-DD', + }, + }; + + const createComponent = (propsData = defaultPropsData) => { wrapper = shallowMount(ExpiresAtField, { - propsData: { - inputAttrs: { - id: 'personal_access_token_expires_at', - name: 'personal_access_token[expires_at]', - placeholder: 'YYYY-MM-DD', - }, - }, + propsData, }); }; diff --git a/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/details_header_spec.js b/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/details_header_spec.js index f4c22d9bfa7..a8d0d15007c 100644 --- a/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/details_header_spec.js +++ b/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/details_header_spec.js @@ -2,6 +2,7 @@ import { GlDropdownItem, GlIcon, GlDropdown } from '@gitlab/ui'; import { shallowMount, createLocalVue } from '@vue/test-utils'; import VueApollo from 'vue-apollo'; import { nextTick } from 'vue'; +import { numberToHumanSize } from '~/lib/utils/number_utils'; import { useFakeDate } from 'helpers/fake_date'; import createMockApollo from 'helpers/mock_apollo_helper'; import { createMockDirective, getBinding } from 'helpers/vue_mock_directive'; @@ -20,7 +21,7 @@ import { ROOT_IMAGE_TEXT, ROOT_IMAGE_TOOLTIP, } from '~/packages_and_registries/container_registry/explorer/constants'; -import getContainerRepositoryTagCountQuery from '~/packages_and_registries/container_registry/explorer/graphql/queries/get_container_repository_tags_count.query.graphql'; +import getContainerRepositoryMetadata from '~/packages_and_registries/container_registry/explorer/graphql/queries/get_container_repository_metadata.query.graphql'; import TitleArea from '~/vue_shared/components/registry/title_area.vue'; import { imageTagsCountMock } from '../../mock_data'; @@ -52,6 +53,7 @@ describe('Details Header', () => { const findDeleteButton = () => wrapper.findComponent(GlDropdownItem); const findInfoIcon = () => wrapper.findComponent(GlIcon); const findMenu = () => wrapper.findComponent(GlDropdown); + const findSize = () => findByTestId('image-size'); const waitForMetadataItems = async () => { // Metadata items are printed by a loop in the title-area and it takes two ticks for them to be available @@ -72,7 +74,7 @@ describe('Details Header', () => { localVue = createLocalVue(); localVue.use(VueApollo); - const requestHandlers = [[getContainerRepositoryTagCountQuery, resolver]]; + const requestHandlers = [[getContainerRepositoryMetadata, resolver]]; apolloProvider = createMockApollo(requestHandlers); } @@ -230,6 +232,30 @@ describe('Details Header', () => { }); }); + describe('size metadata item', () => { + it('when size is not returned, it hides the item', async () => { + mountComponent(); + await waitForMetadataItems(); + + expect(findSize().exists()).toBe(false); + }); + + it('when size is returned shows the item', async () => { + const size = 1000; + mountComponent({ + resolver: jest.fn().mockResolvedValue(imageTagsCountMock({ size })), + }); + + await waitForPromises(); + await waitForMetadataItems(); + + expect(findSize().props()).toMatchObject({ + icon: 'disk', + text: numberToHumanSize(size), + }); + }); + }); + describe('cleanup metadata item', () => { it('has the correct icon', async () => { mountComponent(); diff --git a/spec/frontend/packages_and_registries/container_registry/explorer/mock_data.js b/spec/frontend/packages_and_registries/container_registry/explorer/mock_data.js index 16625d913a5..f0c586fe7f4 100644 --- a/spec/frontend/packages_and_registries/container_registry/explorer/mock_data.js +++ b/spec/frontend/packages_and_registries/container_registry/explorer/mock_data.js @@ -187,6 +187,7 @@ export const imageTagsCountMock = (override) => ({ containerRepository: { id: containerRepositoryMock.id, tagsCount: 13, + size: null, ...override, }, }, diff --git a/spec/frontend/security_configuration/components/training_provider_list_spec.js b/spec/frontend/security_configuration/components/training_provider_list_spec.js index 2683c7665c8..08b79930067 100644 --- a/spec/frontend/security_configuration/components/training_provider_list_spec.js +++ b/spec/frontend/security_configuration/components/training_provider_list_spec.js @@ -25,6 +25,7 @@ import { updateSecurityTrainingProvidersErrorResponse, testProjectPath, testProviderIds, + tempProviderLogos, } from '../mock_data'; Vue.use(VueApollo); @@ -83,6 +84,7 @@ describe('TrainingProviderList component', () => { const findPrimaryProviderRadios = () => wrapper.findAllByTestId('primary-provider-radio'); const findLoader = () => wrapper.findComponent(GlSkeletonLoader); const findErrorAlert = () => wrapper.findComponent(GlAlert); + const findLogos = () => wrapper.findAll('img'); const toggleFirstProvider = () => findFirstToggle().vm.$emit('change', testProviderIds[0]); @@ -179,6 +181,25 @@ describe('TrainingProviderList component', () => { ); }); + describe('provider logo', () => { + beforeEach(async () => { + wrapper.vm.$options.TEMP_PROVIDER_LOGOS = tempProviderLogos; + await waitForQueryToBeLoaded(); + }); + + const providerIndexArray = [0, 1]; + + it.each(providerIndexArray)('displays the correct width for provider %s', (provider) => { + expect(findLogos().at(provider).attributes('width')).toBe('18'); + }); + + it.each(providerIndexArray)('displays the correct svg path for provider %s', (provider) => { + expect(findLogos().at(provider).attributes('src')).toBe( + tempProviderLogos[testProviderIds[provider]].svg, + ); + }); + }); + describe('storing training provider settings', () => { beforeEach(async () => { jest.spyOn(apolloProvider.defaultClient, 'mutate'); diff --git a/spec/frontend/security_configuration/mock_data.js b/spec/frontend/security_configuration/mock_data.js index 8b9730d33da..588fac11987 100644 --- a/spec/frontend/security_configuration/mock_data.js +++ b/spec/frontend/security_configuration/mock_data.js @@ -95,3 +95,14 @@ export const updateSecurityTrainingProvidersErrorResponse = { }, }, }; + +// Will remove once this issue is resolved where the svg path will be available in the GraphQL query +// https://gitlab.com/gitlab-org/gitlab/-/issues/346899 +export const tempProviderLogos = { + [testProviderIds[0]]: { + svg: '/assets/illustrations/vulnerability/vendor-1.svg', + }, + [testProviderIds[1]]: { + svg: '/assets/illustrations/vulnerability/vendor-2.svg', + }, +}; diff --git a/spec/frontend/vue_shared/components/filtered_search_bar/tokens/base_token_spec.js b/spec/frontend/vue_shared/components/filtered_search_bar/tokens/base_token_spec.js index dd9bf2ff598..9684d483c0d 100644 --- a/spec/frontend/vue_shared/components/filtered_search_bar/tokens/base_token_spec.js +++ b/spec/frontend/vue_shared/components/filtered_search_bar/tokens/base_token_spec.js @@ -1,12 +1,23 @@ -import { GlFilteredSearchToken, GlLoadingIcon } from '@gitlab/ui'; +import { + GlFilteredSearchToken, + GlLoadingIcon, + GlFilteredSearchSuggestion, + GlDropdownSectionHeader, + GlDropdownDivider, +} from '@gitlab/ui'; import { mount } from '@vue/test-utils'; import { nextTick } from 'vue'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; import { mockRegularLabel, mockLabels, } from 'jest/vue_shared/components/sidebar/labels_select_vue/mock_data'; -import { DEFAULT_NONE_ANY } from '~/vue_shared/components/filtered_search_bar/constants'; +import { + DEFAULT_NONE_ANY, + OPERATOR_IS, + OPERATOR_IS_NOT, +} from '~/vue_shared/components/filtered_search_bar/constants'; import { getRecentlyUsedSuggestions, setTokenValueToRecentlyUsed, @@ -32,6 +43,7 @@ const defaultStubs = { <div> <slot name="view-token"></slot> <slot name="view"></slot> + <slot name="suggestions"></slot> </div> `, }, @@ -43,6 +55,7 @@ const defaultStubs = { }, }; +const mockSuggestionListTestId = 'suggestion-list'; const defaultSlots = { 'view-token': ` <div class="js-view-token">${mockRegularLabel.title}</div> @@ -52,6 +65,10 @@ const defaultSlots = { `, }; +const defaultScopedSlots = { + 'suggestions-list': `<div data-testid="${mockSuggestionListTestId}" :data-suggestions="JSON.stringify(props.suggestions)"></div>`, +}; + const mockProps = { config: { ...mockLabelToken, recentSuggestionsStorageKey: mockStorageKey }, value: { data: '' }, @@ -62,8 +79,14 @@ const mockProps = { getActiveTokenValue: (labels, data) => labels.find((label) => label.title === data), }; -function createComponent({ props = {}, stubs = defaultStubs, slots = defaultSlots } = {}) { - return mount(BaseToken, { +function createComponent({ + props = {}, + stubs = defaultStubs, + slots = defaultSlots, + scopedSlots = defaultScopedSlots, + mountFn = mount, +} = {}) { + return mountFn(BaseToken, { propsData: { ...mockProps, ...props, @@ -72,9 +95,14 @@ function createComponent({ props = {}, stubs = defaultStubs, slots = defaultSlot portalName: 'fake target', alignSuggestions: jest.fn(), suggestionsListClass: () => 'custom-class', + filteredSearchSuggestionListInstance: { + register: jest.fn(), + unregister: jest.fn(), + }, }, stubs, slots, + scopedSlots, }); } @@ -82,6 +110,9 @@ describe('BaseToken', () => { let wrapper; const findGlFilteredSearchToken = () => wrapper.findComponent(GlFilteredSearchToken); + const findMockSuggestionList = () => wrapper.findByTestId(mockSuggestionListTestId); + const getMockSuggestionListSuggestions = () => + JSON.parse(findMockSuggestionList().attributes('data-suggestions')); afterEach(() => { wrapper.destroy(); @@ -136,6 +167,147 @@ describe('BaseToken', () => { }); }); + describe('suggestions', () => { + describe('with available suggestions', () => { + let mockSuggestions; + + describe.each` + hasSuggestions | searchKey | shouldRenderSuggestions + ${true} | ${null} | ${true} + ${true} | ${'foo'} | ${true} + ${false} | ${null} | ${false} + `( + `when hasSuggestions is $hasSuggestions`, + ({ hasSuggestions, searchKey, shouldRenderSuggestions }) => { + beforeEach(async () => { + mockSuggestions = hasSuggestions ? [{ id: 'Foo' }] : []; + const props = { defaultSuggestions: [], suggestions: mockSuggestions }; + + getRecentlyUsedSuggestions.mockReturnValue([]); + wrapper = createComponent({ props, mountFn: shallowMountExtended, stubs: {} }); + findGlFilteredSearchToken().vm.$emit('input', { data: searchKey }); + + await nextTick(); + }); + + it(`${shouldRenderSuggestions ? 'should' : 'should not'} render suggestions`, () => { + expect(findMockSuggestionList().exists()).toBe(shouldRenderSuggestions); + + if (shouldRenderSuggestions) { + expect(getMockSuggestionListSuggestions()).toEqual(mockSuggestions); + } + }); + }, + ); + }); + + describe('with preloaded suggestions', () => { + const mockPreloadedSuggestions = [{ id: 'Foo' }, { id: 'Bar' }]; + + describe.each` + searchKey | shouldRenderPreloadedSuggestions + ${null} | ${true} + ${'foo'} | ${false} + `('when searchKey is $searchKey', ({ shouldRenderPreloadedSuggestions, searchKey }) => { + beforeEach(async () => { + const props = { preloadedSuggestions: mockPreloadedSuggestions }; + wrapper = createComponent({ props, mountFn: shallowMountExtended, stubs: {} }); + findGlFilteredSearchToken().vm.$emit('input', { data: searchKey }); + + await nextTick(); + }); + + it(`${ + shouldRenderPreloadedSuggestions ? 'should' : 'should not' + } render preloaded suggestions`, () => { + expect(findMockSuggestionList().exists()).toBe(shouldRenderPreloadedSuggestions); + + if (shouldRenderPreloadedSuggestions) { + expect(getMockSuggestionListSuggestions()).toEqual(mockPreloadedSuggestions); + } + }); + }); + }); + + describe('with recent suggestions', () => { + let mockSuggestions; + + describe.each` + searchKey | recentEnabled | shouldRenderRecentSuggestions + ${null} | ${true} | ${true} + ${'foo'} | ${true} | ${false} + ${null} | ${false} | ${false} + `( + 'when searchKey is $searchKey and recentEnabled is $recentEnabled', + ({ shouldRenderRecentSuggestions, recentEnabled, searchKey }) => { + beforeEach(async () => { + const props = { value: { data: '', operator: '=' }, defaultSuggestions: [] }; + + if (recentEnabled) { + mockSuggestions = [{ id: 'Foo' }, { id: 'Bar' }]; + getRecentlyUsedSuggestions.mockReturnValue(mockSuggestions); + } + + props.config = { recentSuggestionsStorageKey: recentEnabled ? mockStorageKey : null }; + + wrapper = createComponent({ props, mountFn: shallowMountExtended, stubs: {} }); + findGlFilteredSearchToken().vm.$emit('input', { data: searchKey }); + + await nextTick(); + }); + + it(`${ + shouldRenderRecentSuggestions ? 'should' : 'should not' + } render recent suggestions`, () => { + expect(findMockSuggestionList().exists()).toBe(shouldRenderRecentSuggestions); + expect(wrapper.findComponent(GlDropdownSectionHeader).exists()).toBe( + shouldRenderRecentSuggestions, + ); + expect(wrapper.findComponent(GlDropdownDivider).exists()).toBe( + shouldRenderRecentSuggestions, + ); + + if (shouldRenderRecentSuggestions) { + expect(getMockSuggestionListSuggestions()).toEqual(mockSuggestions); + } + }); + }, + ); + }); + + describe('with default suggestions', () => { + describe.each` + operator | shouldRenderFilteredSearchSuggestion + ${OPERATOR_IS} | ${true} + ${OPERATOR_IS_NOT} | ${false} + `('when operator is $operator', ({ shouldRenderFilteredSearchSuggestion, operator }) => { + beforeEach(() => { + const props = { + defaultSuggestions: DEFAULT_NONE_ANY, + value: { data: '', operator }, + }; + + wrapper = createComponent({ props, mountFn: shallowMountExtended }); + }); + + it(`${ + shouldRenderFilteredSearchSuggestion ? 'should' : 'should not' + } render GlFilteredSearchSuggestion`, () => { + const filteredSearchSuggestions = wrapper.findAllComponents(GlFilteredSearchSuggestion) + .wrappers; + + if (shouldRenderFilteredSearchSuggestion) { + expect(filteredSearchSuggestions.map((c) => c.props())).toMatchObject( + DEFAULT_NONE_ANY.map((opt) => ({ value: opt.value })), + ); + } else { + expect(filteredSearchSuggestions).toHaveLength(0); + } + }); + }); + }); + }); + describe('methods', () => { describe('handleTokenValueSelected', () => { const mockTokenValue = mockLabels[0]; diff --git a/spec/helpers/container_registry_helper_spec.rb b/spec/helpers/container_registry_helper_spec.rb index 49e56113dd8..57641d4b5df 100644 --- a/spec/helpers/container_registry_helper_spec.rb +++ b/spec/helpers/container_registry_helper_spec.rb @@ -3,25 +3,17 @@ require 'spec_helper' RSpec.describe ContainerRegistryHelper do - using RSpec::Parameterized::TableSyntax - describe '#container_registry_expiration_policies_throttling?' do subject { helper.container_registry_expiration_policies_throttling? } - where(:feature_flag_enabled, :client_support, :expected_result) do - true | true | true - true | false | false - false | true | false - false | false | false - end + it { is_expected.to eq(true) } - with_them do + context 'with container_registry_expiration_policies_throttling disabled' do before do - stub_feature_flags(container_registry_expiration_policies_throttling: feature_flag_enabled) - allow(ContainerRegistry::Client).to receive(:supports_tag_delete?).and_return(client_support) + stub_feature_flags(container_registry_expiration_policies_throttling: false) end - it { is_expected.to eq(expected_result) } + it { is_expected.to eq(false) } end end end diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb index a70ff0bd82d..c1b0f4df29a 100644 --- a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb @@ -104,8 +104,13 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsImporter do .and_yield(pull_request) expect(Gitlab::GithubImport::ImportPullRequestWorker) - .to receive(:perform_async) - .with(project.id, an_instance_of(Hash), an_instance_of(String)) + .to receive(:bulk_perform_in) + .with( + 1.second, + [[project.id, an_instance_of(Hash), an_instance_of(String)]], + batch_delay: 1.minute, + batch_size: 200 + ) waiter = importer.parallel_import diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb index f375e84e0fd..6a19afbc60d 100644 --- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb @@ -22,6 +22,10 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do def collection_method :issues end + + def parallel_import_batch + { size: 10, delay: 1.minute } + end end end @@ -254,35 +258,61 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do describe '#parallel_import' do let(:importer) { importer_class.new(project, client) } + let(:repr_class) { double(:representation) } + let(:worker_class) { double(:worker) } + let(:object) { double(:object) } + let(:batch_size) { 200 } + let(:batch_delay) { 1.minute } - it 'imports data in parallel' do - repr_class = double(:representation) - worker_class = double(:worker) - object = double(:object) - - expect(importer) - .to receive(:each_object_to_import) - .and_yield(object) - - expect(importer) + before do + allow(importer) .to receive(:representation_class) .and_return(repr_class) - expect(importer) + allow(importer) .to receive(:sidekiq_worker_class) .and_return(worker_class) - expect(repr_class) + allow(repr_class) .to receive(:from_api_response) .with(object) .and_return({ title: 'Foo' }) + end + + context 'with multiple objects' do + before do + allow(importer).to receive(:parallel_import_batch) { { size: batch_size, delay: batch_delay } } + expect(importer).to receive(:each_object_to_import).and_yield(object).and_yield(object).and_yield(object) + end - expect(worker_class) - .to receive(:perform_async) - .with(project.id, { title: 'Foo' }, an_instance_of(String)) + it 'imports data in parallel batches with delays' do + expect(worker_class).to receive(:bulk_perform_in).with(1.second, [ + [project.id, { title: 'Foo' }, an_instance_of(String)], + [project.id, { title: 'Foo' }, an_instance_of(String)], + [project.id, { title: 'Foo' }, an_instance_of(String)] + ], batch_size: batch_size, batch_delay: batch_delay) + + importer.parallel_import + end + end - expect(importer.parallel_import) - .to be_an_instance_of(Gitlab::JobWaiter) + context 'when FF is disabled' do + before do + stub_feature_flags(spread_parallel_import: false) + end + + it 'imports data in parallel' do + expect(importer) + .to receive(:each_object_to_import) + .and_yield(object) + + expect(worker_class) + .to receive(:perform_async) + .with(project.id, { title: 'Foo' }, an_instance_of(String)) + + expect(importer.parallel_import) + .to be_an_instance_of(Gitlab::JobWaiter) + end end end diff --git a/spec/lib/gitlab/seeder_spec.rb b/spec/lib/gitlab/seeder_spec.rb index 877461a7064..71d0a41ef98 100644 --- a/spec/lib/gitlab/seeder_spec.rb +++ b/spec/lib/gitlab/seeder_spec.rb @@ -4,6 +4,26 @@ require 'spec_helper' RSpec.describe Gitlab::Seeder do describe '.quiet' do + let(:database_base_models) do + { + main: ApplicationRecord, + ci: Ci::ApplicationRecord + } + end + + it 'disables database logging' do + allow(Gitlab::Database).to receive(:database_base_models) + .and_return(database_base_models.with_indifferent_access) + + described_class.quiet do + expect(ApplicationRecord.logger).to be_nil + expect(Ci::ApplicationRecord.logger).to be_nil + end + + expect(ApplicationRecord.logger).not_to be_nil + expect(Ci::ApplicationRecord.logger).not_to be_nil + end + it 'disables mail deliveries' do expect(ActionMailer::Base.perform_deliveries).to eq(true) diff --git a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb index 833de6ae624..8d46845548a 100644 --- a/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb @@ -122,20 +122,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi it_behaves_like 'sets Redis keys with correct TTL' end - context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do - before do - stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false) - end - - it "does not change the existing wal locations key's TTL" do - expect { duplicate_job.check! } - .to not_change { read_idempotency_key_with_ttl(existing_wal_location_key(idempotency_key, :main)) } - .from([nil, -2]) - .and not_change { read_idempotency_key_with_ttl(existing_wal_location_key(idempotency_key, :ci)) } - .from([nil, -2]) - end - end - it "adds the idempotency key to the jobs payload" do expect { duplicate_job.check! }.to change { job['idempotency_key'] }.from(nil).to(idempotency_key) end @@ -186,28 +172,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi duplicate_job.check! end - context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do - let(:existing_wal) { {} } - - before do - stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false) - end - - it "doesn't call Sidekiq.redis" do - expect(Sidekiq).not_to receive(:redis) - - duplicate_job.update_latest_wal_location! - end - - it "doesn't update a wal location to redis with an offset" do - expect { duplicate_job.update_latest_wal_location! } - .to not_change { read_range_from_redis(wal_location_key(idempotency_key, :main)) } - .from([]) - .and not_change { read_range_from_redis(wal_location_key(idempotency_key, :ci)) } - .from([]) - end - end - context "when the key doesn't exists in redis" do let(:existing_wal) do { @@ -328,20 +292,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi context 'when job is not deduplication and wal locations were not persisted' do it { expect(duplicate_job.latest_wal_locations).to be_empty } end - - context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do - before do - stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false) - end - - it "doesn't call Sidekiq.redis" do - expect(Sidekiq).not_to receive(:redis) - - duplicate_job.latest_wal_locations - end - - it { expect(duplicate_job.latest_wal_locations).to eq({}) } - end end describe '#delete!' do @@ -406,32 +356,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi let(:key) { wal_location_key(idempotency_key, :ci) } let(:from_value) { wal_locations[:ci] } end - - context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do - before do - stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false) - end - - it_behaves_like 'does not delete key from redis', 'latest wal location keys for main database' do - let(:key) { existing_wal_location_key(idempotency_key, :main) } - let(:from_value) { wal_locations[:main] } - end - - it_behaves_like 'does not delete key from redis', 'latest wal location keys for ci database' do - let(:key) { existing_wal_location_key(idempotency_key, :ci) } - let(:from_value) { wal_locations[:ci] } - end - - it_behaves_like 'does not delete key from redis', 'latest wal location keys for main database' do - let(:key) { wal_location_key(idempotency_key, :main) } - let(:from_value) { wal_locations[:main] } - end - - it_behaves_like 'does not delete key from redis', 'latest wal location keys for ci database' do - let(:key) { wal_location_key(idempotency_key, :ci) } - let(:from_value) { wal_locations[:ci] } - end - end end context 'when the idempotency key is not part of the job' do @@ -666,16 +590,6 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi it 'returns true' do expect(duplicate_job).to be_idempotent end - - context 'when preserve_latest_wal_locations_for_idempotent_jobs feature flag is disabled' do - before do - stub_feature_flags(preserve_latest_wal_locations_for_idempotent_jobs: false) - end - - it 'returns false' do - expect(duplicate_job).not_to be_idempotent - end - end end end diff --git a/spec/models/integration_spec.rb b/spec/models/integration_spec.rb index 3d85bd245d9..48d8ba975b6 100644 --- a/spec/models/integration_spec.rb +++ b/spec/models/integration_spec.rb @@ -719,34 +719,63 @@ RSpec.describe Integration do end describe '#api_field_names' do - let(:fake_integration) do - Class.new(Integration) do - def fields - [ - { name: 'token' }, - { name: 'api_token' }, - { name: 'token_api' }, - { name: 'safe_token' }, - { name: 'key' }, - { name: 'api_key' }, - { name: 'password' }, - { name: 'password_field' }, - { name: 'some_safe_field' }, - { name: 'safe_field' }, - { name: 'url' }, - { name: 'trojan_horse', type: 'password' }, - { name: 'trojan_gift', type: 'gift' } - ].shuffle - end + shared_examples 'api field names' do + it 'filters out sensitive fields' do + safe_fields = %w[some_safe_field safe_field url trojan_gift] + + expect(fake_integration.new).to have_attributes( + api_field_names: match_array(safe_fields) + ) end end - it 'filters out sensitive fields' do - safe_fields = %w[some_safe_field safe_field url trojan_gift] + context 'when the class overrides #fields' do + let(:fake_integration) do + Class.new(Integration) do + def fields + [ + { name: 'token' }, + { name: 'api_token' }, + { name: 'token_api' }, + { name: 'safe_token' }, + { name: 'key' }, + { name: 'api_key' }, + { name: 'password' }, + { name: 'password_field' }, + { name: 'some_safe_field' }, + { name: 'safe_field' }, + { name: 'url' }, + { name: 'trojan_horse', type: 'password' }, + { name: 'trojan_gift', type: 'gift' } + ].shuffle + end + end + end - expect(fake_integration.new).to have_attributes( - api_field_names: match_array(safe_fields) - ) + it_behaves_like 'api field names' + end + + context 'when the class uses the field DSL' do + let(:fake_integration) do + Class.new(described_class) do + field :token + field :token + field :api_token + field :token_api + field :safe_token + field :key + field :api_key + field :password + field :password_field + field :some_safe_field + field :safe_field + field :url + field :trojan_horse, type: 'password' + field :trojan_gift, type: 'gift' + end + end + + it_behaves_like 'api field names' end end @@ -921,4 +950,75 @@ RSpec.describe Integration do end end end + + describe 'field DSL' do + let(:integration_type) do + Class.new(described_class) do + field :foo + field :foo_p, storage: :properties + field :foo_dt, storage: :data_fields + + field :bar, type: 'password' + field :password + + field :with_help, + help: -> { 'help' } + + field :a_number, + type: 'number' + end + end + + before do + allow(integration).to receive(:data_fields).and_return(data_fields) + end + + let(:integration) { integration_type.new } + let(:data_fields) { Struct.new(:foo_dt).new } + + it 'checks the value of storage' do + expect do + Class.new(described_class) { field(:foo, storage: 'bar') } + end.to raise_error(ArgumentError, /Unknown field storage/) + end + + it 'provides prop_accessors' do + integration.foo = 1 + expect(integration.foo).to eq 1 + expect(integration.properties['foo']).to eq 1 + expect(integration).to be_foo_changed + + integration.foo_p = 2 + expect(integration.foo_p).to eq 2 + expect(integration.properties['foo_p']).to eq 2 + expect(integration).to be_foo_p_changed + end + + it 'provides data fields' do + integration.foo_dt = 3 + expect(integration.foo_dt).to eq 3 + expect(data_fields.foo_dt).to eq 3 + expect(integration).to be_foo_dt_changed + end + + it 'registers fields in the fields list' do + expect(integration.fields.pluck(:name)).to match_array %w[ + foo foo_p foo_dt bar password with_help a_number + ] + + expect(integration.api_field_names).to match_array %w[ + foo foo_p foo_dt with_help a_number + ] + end + + specify 'fields have expected attributes' do + expect(integration.fields).to include( + have_attributes(name: 'foo', type: 'text'), + have_attributes(name: 'bar', type: 'password'), + have_attributes(name: 'password', type: 'password'), + have_attributes(name: 'a_number', type: 'number'), + have_attributes(name: 'with_help', help: 'help') + ) + end + end end diff --git a/spec/models/integrations/field_spec.rb b/spec/models/integrations/field_spec.rb new file mode 100644 index 00000000000..0d660c4a3ab --- /dev/null +++ b/spec/models/integrations/field_spec.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Integrations::Field do + subject(:field) { described_class.new(**attrs) } + + let(:attrs) { { name: nil } } + + describe '#name' do + before do + attrs[:name] = :foo + end + + it 'is stringified' do + expect(field.name).to eq 'foo' + expect(field[:name]).to eq 'foo' + end + + context 'when not set' do + before do + attrs.delete(:name) + end + + it 'complains' do + expect { field }.to raise_error(ArgumentError) + end + end + end + + described_class::ATTRIBUTES.each do |name| + describe "##{name}" do + it "responds to #{name}" do + expect(field).to be_respond_to(name) + end + + context 'when not set' do + before do + attrs.delete(name) + end + + let(:have_correct_default) do + case name + when :api_only + be false + when :type + eq 'text' + else + be_nil + end + end + + it 'has the correct default' do + expect(field[name]).to have_correct_default + expect(field.send(name)).to have_correct_default + end + end + + context 'when set to a static value' do + before do + attrs[name] = :known + end + + it 'is known' do + expect(field[name]).to eq(:known) + expect(field.send(name)).to eq(:known) + end + end + + context 'when set to a dynamic value' do + before do + attrs[name] = -> { Time.current } + end + + it 'is computed' do + start = Time.current + + travel_to(start + 1.minute) do + expect(field[name]).to be_after(start) + expect(field.send(name)).to be_after(start) + end + end + end + end + end + + describe '#sensitive' do + context 'when empty' do + it { is_expected.not_to be_sensitive } + end + + context 'when a password field' do + before do + attrs[:type] = 'password' + end + + it { is_expected.to be_sensitive } + end + + %w[token api_token api_key secret_key secret_sauce password passphrase].each do |name| + context "when named #{name}" do + before do + attrs[:name] = name + end + + it { is_expected.to be_sensitive } + end + end + + context "when named url" do + before do + attrs[:name] = :url + end + + it { is_expected.not_to be_sensitive } + end + end +end |
