diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-07 09:12:13 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-07 09:12:13 +0000 |
| commit | a7760cee245c8cffc1b322c6031023985e764a12 (patch) | |
| tree | f99e6b6fa4a4e56a1142d8e4e2e2ca1ea5054df0 | |
| parent | 4ea8f39241d94ac4e42bcd44f7b2db80766d2472 (diff) | |
| download | gitlab-ce-a7760cee245c8cffc1b322c6031023985e764a12.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
43 files changed, 492 insertions, 124 deletions
diff --git a/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue b/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue index e9329fb1d88..633dee75237 100644 --- a/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue +++ b/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue @@ -151,6 +151,7 @@ export default { :search-input-placeholder="filteredSearchBar.placeholder" :initial-filter-value="initialFilterValue" data-testid="members-filtered-search-bar" + data-qa-selector="members_filtered_search_bar_content" @onFilter="handleFilter" /> </template> diff --git a/app/assets/stylesheets/page_bundles/merge_requests.scss b/app/assets/stylesheets/page_bundles/merge_requests.scss index 1ee154f9aeb..6c9c648a4b0 100644 --- a/app/assets/stylesheets/page_bundles/merge_requests.scss +++ b/app/assets/stylesheets/page_bundles/merge_requests.scss @@ -132,7 +132,7 @@ $tabs-holder-z-index: 250; .coverage { font-size: 12px; - color: $gray-500; + color: var(--gray-500, $gray-500); line-height: initial; } } @@ -194,12 +194,12 @@ $tabs-holder-z-index: 250; } &:not(:last-child) { - border-bottom: 1px solid $border-color; + border-bottom: 1px solid var(--border-color, $border-color); } } .diff-file-row.is-active { - background-color: $gray-50; + background-color: var(--gray-50, $gray-50); } .mr-conflict-loader { @@ -255,7 +255,7 @@ $tabs-holder-z-index: 250; } .mr-section-container { - border: 1px solid $border-color; + border: 1px solid var(--border-color, $border-color); border-radius: $border-radius-default; background: var(--white, $white); @@ -299,19 +299,19 @@ $tabs-holder-z-index: 250; } .diverged-commits-count { - color: $gl-text-color-secondary; + color: var(--gray-500, $gl-text-color-secondary); } } .mr-state-widget { - color: $gl-text-color; + color: var(--gl-text-color, $gl-text-color); .commit-message-edit { border-radius: $border-radius-default; } .mr-widget-section:not(:first-child) { - border-top: solid 1px $border-color; + border-top: solid 1px var(--border-color, $border-color); } .mr-widget-alert-container + .mr-widget-section { @@ -373,7 +373,7 @@ $tabs-holder-z-index: 250; } .ci-widget { - color: $gl-text-color; + color: var(--gl-text-color, $gl-text-color); display: flex; align-items: center; justify-content: space-between; @@ -422,7 +422,7 @@ $tabs-holder-z-index: 250; .label-branch { @include gl-font-monospace; font-size: 95%; - color: $gl-text-color; + color: var(--gl-text-color, $gl-text-color); font-weight: normal; overflow: hidden; word-break: break-all; @@ -486,7 +486,7 @@ $tabs-holder-z-index: 250; .bold, .gl-font-weight-bold { font-weight: $gl-font-weight-bold; - color: $gray-600; + color: var(--gray-600, $gray-600); margin-left: 10px; } @@ -496,7 +496,7 @@ $tabs-holder-z-index: 250; } .danger { - color: $red-500; + color: var(--red-500, $red-500); } .spacing, @@ -571,10 +571,10 @@ $tabs-holder-z-index: 250; &.mr-pipeline-suggest { border-radius: $border-radius-default; line-height: 20px; - border: 1px solid $border-color; + border: 1px solid var(--border-color, $border-color); .circle-icon-container { - color: $gl-text-color-quaternary; + color: var(--gray-100, $gl-text-color-quaternary); } } } @@ -584,11 +584,11 @@ $tabs-holder-z-index: 250; } .stop-env-container { - color: $gl-text-color; + color: var(--gl-text-color, $gl-text-color); float: right; a { - color: $gl-text-color; + color: var(--gl-text-color, $gl-text-color); } } } @@ -609,15 +609,15 @@ $tabs-holder-z-index: 250; } .mr-widget-border-top { - border-top: 1px solid $border-color; + border-top: 1px solid var(--border-color, $border-color); } .mr-widget-extension { - border-top: 1px solid $border-color; - background-color: $gray-50; + border-top: 1px solid var(--border-color, $border-color); + background-color: var(--gray-50, $gray-50); &.clickable:hover { - background-color: $gray-100; + background-color: var(--gray-100, $gray-100); cursor: pointer; } } @@ -638,7 +638,7 @@ $tabs-holder-z-index: 250; .mr-widget-heading { position: relative; - border: 1px solid $border-color; + border: 1px solid var(--border-color, $border-color); border-radius: $border-radius-default; background: var(--white, $white); @@ -668,7 +668,7 @@ $tabs-holder-z-index: 250; &::before { content: ''; - border-left: 1px solid $gray-100; + border-left: 1px solid var(--gray-100, $gray-100); position: absolute; left: 28px; top: -17px; @@ -679,8 +679,8 @@ $tabs-holder-z-index: 250; .mr-version-controls { position: relative; z-index: $tabs-holder-z-index + 10; - background: $white; - color: $gl-text-color; + background: var(--white, $white); + color: var(--gl-text-color, $gl-text-color); margin-top: -1px; .mr-version-menus-container { @@ -713,7 +713,7 @@ $tabs-holder-z-index: 250; } .dropdown-title { - color: $gl-text-color; + color: var(--gl-text-color, $gl-text-color); } // Shortening button height by 1px to make compare-versions diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index db82d5bbf29..ebda5872f1c 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -46,17 +46,39 @@ class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel .to_a end - def self.mark_records_processed(all_records) - # Run a query for each partition to optimize the row lookup by primary key (partition, id) + def self.mark_records_processed(records) + update_by_partition(records) do |partitioned_scope| + partitioned_scope.update_all(status: :processed) + end + end + + def self.reschedule(records, consume_after) + update_by_partition(records) do |partitioned_scope| + partitioned_scope.update_all(consume_after: consume_after, cleanup_attempts: 0) + end + end + + def self.increment_attempts(records) + update_by_partition(records) do |partitioned_scope| + # Naive incrementing of the cleanup_attempts is good enough for us. + partitioned_scope.update_all('cleanup_attempts = cleanup_attempts + 1') + end + end + + def self.update_by_partition(records) update_count = 0 - all_records.group_by(&:partition_number).each do |partition, records_within_partition| - update_count += status_pending + # Run a query for each partition to optimize the row lookup by primary key (partition, id) + records.group_by(&:partition_number).each do |partition, records_within_partition| + partitioned_scope = status_pending .for_partition(partition) .where(id: records_within_partition.pluck(:id)) - .update_all(status: :processed) + + update_count += yield(partitioned_scope) end update_count end + + private_class_method :update_by_partition end diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index cfc993efd9a..fed86c020dd 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -12,7 +12,7 @@ module Namespaces def as_ids return super unless use_traversal_ids? - select('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id') + select(Arel.sql('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)]').as('id')) end def roots diff --git a/app/services/loose_foreign_keys/batch_cleaner_service.rb b/app/services/loose_foreign_keys/batch_cleaner_service.rb index de52cbba576..f3db2037911 100644 --- a/app/services/loose_foreign_keys/batch_cleaner_service.rb +++ b/app/services/loose_foreign_keys/batch_cleaner_service.rb @@ -2,6 +2,9 @@ module LooseForeignKeys class BatchCleanerService + CLEANUP_ATTEMPTS_BEFORE_RESCHEDULE = 3 + CONSUME_AFTER_RESCHEDULE = 5.minutes + def initialize(parent_table:, loose_foreign_key_definitions:, deleted_parent_records:, modification_tracker: LooseForeignKeys::ModificationTracker.new) @parent_table = parent_table @loose_foreign_key_definitions = loose_foreign_key_definitions @@ -11,15 +14,31 @@ module LooseForeignKeys :loose_foreign_key_processed_deleted_records, 'The number of processed loose foreign key deleted records' ) + @deleted_records_rescheduled_count = Gitlab::Metrics.counter( + :loose_foreign_key_rescheduled_deleted_records, + 'The number of rescheduled loose foreign key deleted records' + ) + @deleted_records_incremented_count = Gitlab::Metrics.counter( + :loose_foreign_key_incremented_deleted_records, + 'The number of loose foreign key deleted records with incremented cleanup_attempts' + ) end def execute loose_foreign_key_definitions.each do |loose_foreign_key_definition| run_cleaner_service(loose_foreign_key_definition, with_skip_locked: true) - break if modification_tracker.over_limit? + + if modification_tracker.over_limit? + handle_over_limit + break + end run_cleaner_service(loose_foreign_key_definition, with_skip_locked: false) - break if modification_tracker.over_limit? + + if modification_tracker.over_limit? + handle_over_limit + break + end end return if modification_tracker.over_limit? @@ -27,12 +46,33 @@ module LooseForeignKeys # At this point, all associations are cleaned up, we can update the status of the parent records update_count = LooseForeignKeys::DeletedRecord.mark_records_processed(deleted_parent_records) - deleted_records_counter.increment({ table: parent_table, db_config_name: LooseForeignKeys::DeletedRecord.connection.pool.db_config.name }, update_count) + deleted_records_counter.increment({ table: parent_table, db_config_name: db_config_name }, update_count) end private - attr_reader :parent_table, :loose_foreign_key_definitions, :deleted_parent_records, :modification_tracker, :deleted_records_counter + attr_reader :parent_table, :loose_foreign_key_definitions, :deleted_parent_records, :modification_tracker, :deleted_records_counter, :deleted_records_rescheduled_count, :deleted_records_incremented_count + + def handle_over_limit + return if Feature.disabled?(:lfk_fair_queueing) + + records_to_reschedule = [] + records_to_increment = [] + + deleted_parent_records.each do |deleted_record| + if deleted_record.cleanup_attempts >= CLEANUP_ATTEMPTS_BEFORE_RESCHEDULE + records_to_reschedule << deleted_record + else + records_to_increment << deleted_record + end + end + + reschedule_count = LooseForeignKeys::DeletedRecord.reschedule(records_to_reschedule, CONSUME_AFTER_RESCHEDULE.from_now) + deleted_records_rescheduled_count.increment({ table: parent_table, db_config_name: db_config_name }, reschedule_count) + + increment_count = LooseForeignKeys::DeletedRecord.increment_attempts(records_to_increment) + deleted_records_incremented_count.increment({ table: parent_table, db_config_name: db_config_name }, increment_count) + end def record_result(cleaner, result) if cleaner.async_delete? @@ -60,5 +100,9 @@ module LooseForeignKeys end end end + + def db_config_name + LooseForeignKeys::DeletedRecord.connection.pool.db_config.name + end end end diff --git a/app/views/projects/settings/ci_cd/_form.html.haml b/app/views/projects/settings/ci_cd/_form.html.haml index d3cdfc4f7c9..c70e153ae41 100644 --- a/app/views/projects/settings/ci_cd/_form.html.haml +++ b/app/views/projects/settings/ci_cd/_form.html.haml @@ -94,7 +94,7 @@ .input-group-text / %p.form-text.text-muted = html_escape(_('The regular expression used to find test coverage output in the job log. For example, use %{regex} for Simplecov (Ruby). Leave blank to disable.')) % { regex: '<code>\(\d+.\d+%\)</code>'.html_safe } - = link_to sprite_icon('question-o'), help_page_path('ci/pipelines/settings', anchor: 'add-test-coverage-results-to-a-merge-request'), target: '_blank', rel: 'noopener noreferrer' + = link_to sprite_icon('question-o'), help_page_path('ci/pipelines/settings', anchor: 'add-test-coverage-results-to-a-merge-request-deprecated'), target: '_blank', rel: 'noopener noreferrer' = f.submit _('Save changes'), class: "btn gl-button btn-confirm", data: { qa_selector: 'save_general_pipelines_changes_button' } diff --git a/config/feature_flags/development/deployment_approvals.yml b/config/feature_flags/development/lfk_fair_queueing.yml index 5083ccd28bf..ac67ffa14f0 100644 --- a/config/feature_flags/development/deployment_approvals.yml +++ b/config/feature_flags/development/lfk_fair_queueing.yml @@ -1,8 +1,8 @@ --- -name: deployment_approvals -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74932 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/347342 -milestone: '14.6' +name: lfk_fair_queueing +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79116 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/351082 +milestone: '14.8' type: development -group: group::release +group: group::sharding default_enabled: false diff --git a/data/deprecations/14-9-deprecate-testcoveragesetting.yml b/data/deprecations/14-9-deprecate-testcoveragesetting.yml new file mode 100644 index 00000000000..19bc6004b02 --- /dev/null +++ b/data/deprecations/14-9-deprecate-testcoveragesetting.yml @@ -0,0 +1,14 @@ +- name: "Test coverage project CI/CD setting" # The name of the feature to be deprecated + announcement_milestone: "14.8" # The milestone when this feature was first announced as deprecated. + announcement_date: "2022-02-22" # The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post. + removal_milestone: "15.0" # The milestone when this feature is planned to be removed + removal_date: "2022-05-22" # The date of the milestone release when this feature is planned to be removed. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post. + breaking_change: true # If this deprecation is a breaking change, set this value to true + reporter: jreporter # GitLab username of the person reporting the deprecation + body: | # Do not modify this line, instead modify the lines below. + To simplify setting a test coverage pattern, in GitLab 15.0 the + [project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-to-a-merge-request-deprecated) + is being removed. + + Instead, using the project’s `.gitlab-ci.yml`, provide a regular expression with the `coverage` keyword to set + testing coverage results in merge requests. diff --git a/db/migrate/20220125084127_add_cleanup_attempts_to_loose_foreign_keys_deleted_records.rb b/db/migrate/20220125084127_add_cleanup_attempts_to_loose_foreign_keys_deleted_records.rb new file mode 100644 index 00000000000..e0c80ad79e5 --- /dev/null +++ b/db/migrate/20220125084127_add_cleanup_attempts_to_loose_foreign_keys_deleted_records.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddCleanupAttemptsToLooseForeignKeysDeletedRecords < Gitlab::Database::Migration[1.0] + enable_lock_retries! + + def up + add_column :loose_foreign_keys_deleted_records, :cleanup_attempts, :smallint, default: 0 + end + + def down + remove_column :loose_foreign_keys_deleted_records, :cleanup_attempts + end +end diff --git a/db/schema_migrations/20220125084127 b/db/schema_migrations/20220125084127 new file mode 100644 index 00000000000..f9ba7a4628b --- /dev/null +++ b/db/schema_migrations/20220125084127 @@ -0,0 +1 @@ +c50a21430e52fc6ccbe7ab4aba99ae3a48d1c69858f7886da115f54e19fc27ea
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index dad5942f213..6dc825fdedb 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -279,6 +279,7 @@ CREATE TABLE loose_foreign_keys_deleted_records ( created_at timestamp with time zone DEFAULT now() NOT NULL, fully_qualified_table_name text NOT NULL, consume_after timestamp with time zone DEFAULT now(), + cleanup_attempts smallint DEFAULT 0, CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150)) ) PARTITION BY LIST (partition); diff --git a/doc/api/deployments.md b/doc/api/deployments.md index 7fc870006ac..29f9bd3e2ee 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -448,7 +448,10 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a ## Approve or reject a blocked deployment **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343864) in GitLab 14.7 [with a flag](../administration/feature_flags.md) named `deployment_approvals`. Disabled by default. This feature is not ready for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343864) in GitLab 14.7 [with a flag](../administration/feature_flags.md) named `deployment_approvals`. Disabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/347342) in GitLab 14.8. + +See [Deployment Approvals](../ci/environments/deployment_approvals.md) for more information about this feature. ```plaintext POST /projects/:id/deployments/:deployment_id/approval diff --git a/doc/ci/environments/deployment_approvals.md b/doc/ci/environments/deployment_approvals.md index a82da42d24a..d4550f1da97 100644 --- a/doc/ci/environments/deployment_approvals.md +++ b/doc/ci/environments/deployment_approvals.md @@ -7,7 +7,8 @@ description: Require approvals prior to deploying to a Protected Environment # Deployment approvals **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343864) in GitLab 14.7 with a flag named `deployment_approvals`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343864) in GitLab 14.7 with a flag named `deployment_approvals`. Disabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/347342) in GitLab 14.8. WARNING: This feature is in an alpha stage and subject to change without prior notice. diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 7db57fc44ff..e22746dbfa0 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -195,7 +195,13 @@ Jobs that exceed the timeout are marked as failed. You can override this value [for individual runners](../runners/configure_runners.md#set-maximum-job-timeout-for-a-runner). -## Add test coverage results to a merge request +## Add test coverage results to a merge request (DEPRECATED) + +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 14.9. Replaced by [`coverage` keyword](../yaml/index.md#coverage). + +WARNING: +This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) +for use in GitLab 14.9, and is planned for [removal](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 15.0. If you use test coverage in your code, you can use a regular expression to find coverage results in the job log. You can then include these results @@ -358,7 +364,7 @@ https://gitlab.example.com/<namespace>/<project>/badges/<branch>/pipeline.svg?ig ### Test coverage report badge -You can define the regular expression for the [coverage report](#add-test-coverage-results-to-a-merge-request) +You can define the regular expression for the [coverage report](#add-test-coverage-results-to-a-merge-request-deprecated) that each job log is matched against. This means that each job in the pipeline can have the test coverage percentage value defined. diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index c4c897571e7..99e4536e94f 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -1349,7 +1349,7 @@ In this example: **Additional details**: - Coverage regular expressions set in `gitlab-ci.yml` take precedence over coverage regular expression set in the - [GitLab UI](../pipelines/settings.md#add-test-coverage-results-to-a-merge-request). + [GitLab UI](../pipelines/settings.md#add-test-coverage-results-to-a-merge-request-deprecated). - If there is more than one matched line in the job output, the last line is used (the first result of reverse search). - If there are multiple matches in a single line, the last match is searched diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 1e4d13a0039..77a6311f3c2 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -853,6 +853,23 @@ existing runners. **Planned removal milestone: 15.0 (2022-05-22)** +### Test coverage project CI/CD setting + +WARNING: +This feature will be changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +To simplify setting a test coverage pattern, in GitLab 15.0 the +[project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-to-a-merge-request-deprecated) +is being removed. + +Instead, using the project’s `.gitlab-ci.yml`, provide a regular expression with the `coverage` keyword to set +testing coverage results in merge requests. + +**Planned removal milestone: 15.0 (2022-05-22)** + ### Vulnerability Check WARNING: diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index c67bd3de8a6..728d28d293c 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -53,7 +53,7 @@ from any job in any stage in the pipeline. The coverage displays for each line: Hovering over the coverage bar provides further information, such as the number of times the line was checked by tests. -Uploading a test coverage report does not enable [test coverage results in Merge Requests](../../../ci/pipelines/settings.md#add-test-coverage-results-to-a-merge-request) or [code coverage history](../../../ci/pipelines/settings.md#view-code-coverage-history). You must configure these separately. +Uploading a test coverage report does not enable [test coverage results in Merge Requests](../../../ci/pipelines/settings.md#add-test-coverage-results-to-a-merge-request-deprecated) or [code coverage history](../../../ci/pipelines/settings.md#view-code-coverage-history). You must configure these separately. ### Limits diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb index 95afd5a8595..58963779c69 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb @@ -12,6 +12,7 @@ module Gitlab array_scope_table = Arel::Table.new(ARRAY_SCOPE_CTE_NAME) @columns = columns.map do |column| + column = column.right if column.is_a?(Arel::Nodes::As) ColumnData.new(column, "array_scope_#{column}", array_scope_table) end end diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index 1e50df5f21c..53a0badfc30 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -28,23 +28,31 @@ module Gitlab end def api_request? - path.start_with?('/api') + logical_path.start_with?('/api') + end + + def logical_path + @logical_path ||= path.delete_prefix(Gitlab.config.gitlab.relative_url_root) + end + + def matches?(regex) + logical_path.match?(regex) end def api_internal_request? - path.match?(%r{^/api/v\d+/internal/}) + matches?(%r{^/api/v\d+/internal/}) end def health_check_request? - path.match?(%r{^/-/(health|liveness|readiness|metrics)}) + matches?(%r{^/-/(health|liveness|readiness|metrics)}) end def container_registry_event? - path.match?(%r{^/api/v\d+/container_registry_event/}) + matches?(%r{^/api/v\d+/container_registry_event/}) end def product_analytics_collector_request? - path.start_with?('/-/collector/i') + logical_path.start_with?('/-/collector/i') end def should_be_skipped? @@ -56,7 +64,7 @@ module Gitlab end def protected_path? - path.match?(protected_paths_regex) + matches?(protected_paths_regex) end def throttle?(throttle, authenticated:) @@ -178,15 +186,15 @@ module Gitlab end def packages_api_path? - path.match?(::Gitlab::Regex::Packages::API_PATH_REGEX) + matches?(::Gitlab::Regex::Packages::API_PATH_REGEX) end def git_lfs_path? - path.match?(::Gitlab::PathRegex.repository_git_lfs_route_regex) + matches?(::Gitlab::PathRegex.repository_git_lfs_route_regex) end def files_api_path? - path.match?(FILES_PATH_REGEX) + matches?(FILES_PATH_REGEX) end def frontend_request? @@ -206,7 +214,7 @@ module Gitlab with_projects = params['with_projects'] with_projects = true if with_projects.blank? - path.match?(GROUP_PATH_REGEX) && Gitlab::Utils.to_boolean(with_projects) + matches?(GROUP_PATH_REGEX) && Gitlab::Utils.to_boolean(with_projects) end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 46c5e766830..279a76bbfdb 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -31974,9 +31974,6 @@ msgstr "" msgid "SecurityOrchestration|Description" msgstr "" -msgid "SecurityOrchestration|Disabled" -msgstr "" - msgid "SecurityOrchestration|Edit policy" msgstr "" @@ -32010,6 +32007,9 @@ msgstr "" msgid "SecurityOrchestration|No rules defined - policy will not run." msgstr "" +msgid "SecurityOrchestration|Not enabled" +msgstr "" + msgid "SecurityOrchestration|Only owners can update Security Policy Project" msgstr "" diff --git a/qa/Gemfile b/qa/Gemfile index c07f9dc96a7..5373349d0e8 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -20,6 +20,7 @@ gem 'parallel_tests', '~> 2.29' gem 'rotp', '~> 3.1.0' gem 'timecop', '~> 0.9.1' gem 'parallel', '~> 1.19' +gem 'rainbow', '~> 3.0.0' gem 'rspec-parameterized', '~> 0.4.2' gem 'octokit', '~> 4.21' gem 'webdrivers', '~> 5.0' diff --git a/qa/qa/page/component/members_filter.rb b/qa/qa/page/component/members_filter.rb new file mode 100644 index 00000000000..ac07fe7e9fa --- /dev/null +++ b/qa/qa/page/component/members_filter.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module QA + module Page + module Component + module MembersFilter + extend QA::Page::PageConcern + + def self.included(base) + super + + base.view 'app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue' do + element :members_filtered_search_bar_content + end + end + + def search_member(username) + # TODO: Update the two actions below to use direct qa selectors once this is implemented: + # https://gitlab.com/gitlab-org/gitlab-ui/-/issues/1688 + find_element(:members_filtered_search_bar_content).find('input').set(username) + find('.gl-search-box-by-click-search-button').click + end + end + end + end +end diff --git a/qa/qa/page/group/members.rb b/qa/qa/page/group/members.rb index ccc901932f4..c80bdadb11f 100644 --- a/qa/qa/page/group/members.rb +++ b/qa/qa/page/group/members.rb @@ -6,6 +6,7 @@ module QA class Members < Page::Base include Page::Component::InviteMembersModal include Page::Component::UsersSelect + include Page::Component::MembersFilter view 'app/assets/javascripts/members/components/modals/remove_member_modal.vue' do element :remove_member_modal_content @@ -31,6 +32,8 @@ module QA end def update_access_level(username, access_level) + search_member(username) + within_element(:member_row, text: username) do click_element :access_level_dropdown click_element :access_level_link, text: access_level diff --git a/qa/qa/page/project/members.rb b/qa/qa/page/project/members.rb index 1102abd6646..30748ed920b 100644 --- a/qa/qa/page/project/members.rb +++ b/qa/qa/page/project/members.rb @@ -5,6 +5,7 @@ module QA module Project class Members < Page::Base include QA::Page::Component::InviteMembersModal + include QA::Page::Component::MembersFilter view 'app/assets/javascripts/members/components/members_tabs.vue' do element :groups_list_tab diff --git a/qa/qa/resource/api_fabricator.rb b/qa/qa/resource/api_fabricator.rb index 4c77c515cfd..1958884916c 100644 --- a/qa/qa/resource/api_fabricator.rb +++ b/qa/qa/resource/api_fabricator.rb @@ -7,11 +7,11 @@ module QA module Resource module ApiFabricator include Capybara::DSL + include Support::API include Errors - attr_reader :api_resource, :api_response attr_writer :api_client - attr_accessor :api_user + attr_accessor :api_user, :api_resource, :api_response def api_support? respond_to?(:api_get_path) && @@ -48,9 +48,6 @@ module QA end end - include Support::API - attr_writer :api_resource, :api_response - def api_put(body = api_put_body) response = put( Runtime::API::Request.new(api_client, api_put_path).url, @@ -67,6 +64,16 @@ module QA @api_fabrication_http_method ||= :post end + # Checks if a resource already exists + # + # @return [Boolean] true if the resource returns HTTP status code 200 + def exists? + request = Runtime::API::Request.new(api_client, api_get_path) + response = get(request.url) + + response.code == HTTP_STATUS_OK + end + private def resource_web_url(resource) diff --git a/qa/qa/resource/base.rb b/qa/qa/resource/base.rb index f8522872ddf..182a210d4ea 100644 --- a/qa/qa/resource/base.rb +++ b/qa/qa/resource/base.rb @@ -8,6 +8,7 @@ module QA class Base include ApiFabricator extend Capybara::DSL + using Rainbow NoValueError = Class.new(RuntimeError) @@ -102,7 +103,7 @@ module QA Runtime::Logger.debug do msg = ["==#{'=' * parents.size}>"] - msg << "#{fabrication_http_method} a #{name}" + msg << "#{fabrication_http_method} a #{Rainbow(name).black.bg(:white)}" msg << identifier msg << "as a dependency of #{parents.last}" if parents.any? msg << "via #{fabrication_method}" @@ -182,7 +183,7 @@ module QA end def visit!(skip_resp_code_check: false) - Runtime::Logger.debug(%(Visiting #{self.class.name} at "#{web_url}")) + Runtime::Logger.debug("Visiting #{Rainbow(self.class.name).black.bg(:white)} at #{web_url}") # Just in case an async action is not yet complete Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check) diff --git a/qa/qa/resource/clusters/agent.rb b/qa/qa/resource/clusters/agent.rb index ee5a292b9b3..b190634f357 100644 --- a/qa/qa/resource/clusters/agent.rb +++ b/qa/qa/resource/clusters/agent.rb @@ -17,7 +17,6 @@ module QA end def fabricate! - puts 'TODO: FABRICATE VIA UI' end def resource_web_url(resource) diff --git a/qa/qa/resource/clusters/agent_token.rb b/qa/qa/resource/clusters/agent_token.rb index 6d803b94564..c1cf5c2f37b 100644 --- a/qa/qa/resource/clusters/agent_token.rb +++ b/qa/qa/resource/clusters/agent_token.rb @@ -11,7 +11,6 @@ module QA end def fabricate! - puts 'TODO: FABRICATE VIA UI' end def resource_web_url(resource) diff --git a/qa/qa/resource/reusable.rb b/qa/qa/resource/reusable.rb index 24b0a1f6bce..02316a045c7 100644 --- a/qa/qa/resource/reusable.rb +++ b/qa/qa/resource/reusable.rb @@ -46,7 +46,9 @@ module QA # @return [Hash<Symbol, QA::Resource>] the resources that were to be removed. def remove_all_via_api! resources.each do |reuse_as, resource| - QA::Runtime::Logger.debug("#{self.name} - removing #{reuse_as}") + QA::Runtime::Logger.debug("#{self.name} - removing resource reused as :#{reuse_as}") + next QA::Runtime::Logger.debug("#{self.name} reused as :#{reuse_as} has already been removed.") unless resource.exists? + resource.method(:remove_via_api!).super_method.call end end diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index cb4d1418145..3e442814734 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -65,6 +65,10 @@ module QA ENV['QA_LOG_PATH'] || $stdout end + def colorized_logs? + enabled?(ENV['COLORIZED_LOGS'], default: false) + end + # set to 'false' to have the browser run visibly instead of headless def webdriver_headless? if ENV.key?('CHROME_HEADLESS') diff --git a/qa/qa/runtime/logger.rb b/qa/qa/runtime/logger.rb index a70c8faf7d2..81c41000033 100644 --- a/qa/qa/runtime/logger.rb +++ b/qa/qa/runtime/logger.rb @@ -2,11 +2,13 @@ require 'logger' require 'forwardable' +require 'rainbow/refinement' module QA module Runtime module Logger extend SingleForwardable + using Rainbow def_delegators :logger, :debug, :info, :warn, :error, :fatal, :unknown @@ -14,8 +16,16 @@ module QA attr_writer :logger def logger + Rainbow.enabled = Runtime::Env.colorized_logs? + @logger ||= ::Logger.new(Runtime::Env.log_destination).tap do |logger| logger.level = Runtime::Env.debug? ? ::Logger::DEBUG : ::Logger::ERROR + + logger.formatter = proc do |severity, datetime, progname, msg| + date_format = datetime.strftime("%Y-%m-%d %H:%M:%S") + + "[date=#{date_format} from=QA Tests] #{severity.ljust(5)} -- ".yellow + "#{msg}\n" + end end end end diff --git a/qa/qa/service/cluster_provider/gcloud.rb b/qa/qa/service/cluster_provider/gcloud.rb index c6d1f6cfe88..77677745f7a 100644 --- a/qa/qa/service/cluster_provider/gcloud.rb +++ b/qa/qa/service/cluster_provider/gcloud.rb @@ -49,7 +49,7 @@ module QA if account.empty? raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally." else - puts "gcloud account found. Using: #{account} for creating K8s cluster." + QA::Runtime::Logger.debug("gcloud account found. Using: #{account} for creating K8s cluster.") end end end diff --git a/qa/qa/service/shellout.rb b/qa/qa/service/shellout.rb index 33f2bc6e255..33d1d10b515 100644 --- a/qa/qa/service/shellout.rb +++ b/qa/qa/service/shellout.rb @@ -5,6 +5,7 @@ require 'open3' module QA module Service module Shellout + using Rainbow CommandError = Class.new(StandardError) module_function @@ -14,7 +15,7 @@ module QA # as a library - gitlab-org/gitlab-qa#94 # def shell(command, stdin_data: nil, fail_on_exception: true) - puts "Executing `#{command}`" + QA::Runtime::Logger.info("Executing `#{command}`".cyan) Open3.popen2e(*command) do |stdin, out, wait| stdin.puts(stdin_data) if stdin_data diff --git a/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb b/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb index 6a31d173440..fe6c89f4ee4 100644 --- a/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb +++ b/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb @@ -31,44 +31,50 @@ module QA end it 'is not allowed to push code via the CLI', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347863' do - expect do - Resource::Repository::Push.fabricate! do |push| - push.repository_http_uri = @project.repository_http_location.uri - push.file_name = 'test.txt' - push.file_content = "# This is a test project named #{@project.name}" - push.commit_message = 'Add test.txt' - push.branch_name = "new_branch_#{SecureRandom.hex(8)}" - push.user = @user - end - end.to raise_error(QA::Support::Run::CommandError, /You are not allowed to push code to this project/) + QA::Support::Retrier.retry_on_exception(max_attempts: 5, sleep_interval: 2) do + expect do + Resource::Repository::Push.fabricate! do |push| + push.repository_http_uri = @project.repository_http_location.uri + push.file_name = 'test.txt' + push.file_content = "# This is a test project named #{@project.name}" + push.commit_message = 'Add test.txt' + push.branch_name = "new_branch_#{SecureRandom.hex(8)}" + push.user = @user + end + end.to raise_error(QA::Support::Run::CommandError, /You are not allowed to push code to this project/) + end end it 'is not allowed to create a file via the API', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347864' do - expect do - Resource::File.fabricate_via_api! do |file| - file.api_client = @user_api_client - file.project = @project - file.branch = "new_branch_#{SecureRandom.hex(8)}" - file.commit_message = 'Add new file' - file.name = 'test.txt' - file.content = "New file" - end - end.to raise_error(Resource::ApiFabricator::ResourceFabricationFailedError, /403 Forbidden/) + QA::Support::Retrier.retry_on_exception(max_attempts: 5, sleep_interval: 2) do + expect do + Resource::File.fabricate_via_api! do |file| + file.api_client = @user_api_client + file.project = @project + file.branch = "new_branch_#{SecureRandom.hex(8)}" + file.commit_message = 'Add new file' + file.name = 'test.txt' + file.content = "New file" + end + end.to raise_error(Resource::ApiFabricator::ResourceFabricationFailedError, /403 Forbidden/) + end end it 'is not allowed to commit via the API', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347865' do - expect do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.api_client = @user_api_client - commit.project = @project - commit.branch = "new_branch_#{SecureRandom.hex(8)}" - commit.start_branch = @project.default_branch - commit.commit_message = 'Add new file' - commit.add_files([ - { file_path: 'test.txt', content: 'new file' } - ]) - end - end.to raise_error(Resource::ApiFabricator::ResourceFabricationFailedError, /403 Forbidden - You are not allowed to push into this branch/) + QA::Support::Retrier.retry_on_exception(max_attempts: 5, sleep_interval: 2) do + expect do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.api_client = @user_api_client + commit.project = @project + commit.branch = "new_branch_#{SecureRandom.hex(8)}" + commit.start_branch = @project.default_branch + commit.commit_message = 'Add new file' + commit.add_files([ + { file_path: 'test.txt', content: 'new file' } + ]) + end + end.to raise_error(Resource::ApiFabricator::ResourceFabricationFailedError, /403 Forbidden - You are not allowed to push into this branch/) + end end end diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb index 895027a588d..bfb810b5c2b 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Manage', :requires_admin, quarantine: { - issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/350598', - type: :needs_update, - only: { subdomain: :staging } - } do + RSpec.describe 'Manage', :requires_admin do describe 'Add project member' do before do Runtime::Feature.enable(:invite_members_group_modal) @@ -25,7 +21,7 @@ module QA Page::Project::Menu.perform(&:click_members) Page::Project::Members.perform do |members| members.add_member(user.username) - + members.search_member(user.username) expect(members).to have_content("@#{user.username}") end end diff --git a/qa/qa/support/page/logging.rb b/qa/qa/support/page/logging.rb index f5299ed840d..b402639b843 100644 --- a/qa/qa/support/page/logging.rb +++ b/qa/qa/support/page/logging.rb @@ -4,6 +4,8 @@ module QA module Support module Page module Logging + using Rainbow + def assert_no_element(name) log("asserting no element :#{name}") @@ -17,7 +19,7 @@ module QA end def scroll_to(selector, text: nil) - msg = "scrolling to :#{selector}" + msg = "scrolling to :#{Rainbow(selector).underline.bright}" msg += " with text: #{text}" if text log(msg) @@ -37,7 +39,7 @@ module QA element = super - log("found :#{name}") if element + log("found :#{Rainbow(name).underline.bright}") element end @@ -47,7 +49,7 @@ module QA elements = super - log("found #{elements.size} :#{name}") if elements + log("found #{elements.size} :#{Rainbow(name).underline.bright}") if elements elements end @@ -71,7 +73,7 @@ module QA end def click_element(name, page = nil, **kwargs) - msg = ["clicking :#{name}"] + msg = ["clicking :#{Rainbow(name).underline.bright}"] msg << ", expecting to be at #{page.class}" if page msg << "with args #{kwargs}" @@ -170,7 +172,7 @@ module QA end def log_has_element_or_not(method, name, found, **kwargs) - msg = ["#{method} :#{name}"] + msg = ["#{method} :#{Rainbow(name).underline.bright}"] msg << %Q(with text "#{kwargs[:text]}") if kwargs[:text] msg << "class: #{kwargs[:class]}" if kwargs[:class] msg << "(wait: #{kwargs[:wait] || Capybara.default_max_wait_time})" diff --git a/qa/qa/support/repeater.rb b/qa/qa/support/repeater.rb index a4e8035f964..1b9aa809051 100644 --- a/qa/qa/support/repeater.rb +++ b/qa/qa/support/repeater.rb @@ -1,10 +1,11 @@ # frozen_string_literal: true - require 'active_support/inflector' +require 'rainbow/refinement' module QA module Support module Repeater + using Rainbow DEFAULT_MAX_WAIT_TIME = 60 RepeaterConditionExceededError = Class.new(RuntimeError) @@ -39,7 +40,7 @@ module QA QA::Runtime::Logger.debug(msg.join(' ')) end - QA::Runtime::Logger.debug("Attempt number #{attempts + 1}") if log && max_attempts && attempts > 0 + QA::Runtime::Logger.debug("Attempt number #{attempts + 1}".bg(:yellow).black) if log && max_attempts && attempts > 0 result = yield if result diff --git a/qa/qa/tools/initialize_gitlab_auth.rb b/qa/qa/tools/initialize_gitlab_auth.rb index 86791f1f624..18e90f0d739 100644 --- a/qa/qa/tools/initialize_gitlab_auth.rb +++ b/qa/qa/tools/initialize_gitlab_auth.rb @@ -16,7 +16,7 @@ module QA def run Runtime::Scenario.define(:gitlab_address, address) - puts "Signing in and creating the default password for the root user if it's not set already..." + QA::Runtime::Logger.info("Signing in and creating the default password for the root user if it's not set already...") QA::Runtime::Browser.visit(:gitlab, QA::Page::Main::Login) Flow::Login.sign_in diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 47791f76970..3be71d51a81 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -6,6 +6,7 @@ require 'securerandom' require 'pathname' require 'active_support/core_ext/hash' require 'active_support/core_ext/object/blank' +require 'rainbow/refinement' require_relative 'qa_deprecation_toolkit_env' QaDeprecationToolkitEnv.configure! diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb index 2cebf0d9473..087bfb197ec 100644 --- a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb +++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb @@ -16,4 +16,13 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::ArrayScopeCol it { expect { array_scope_columns }.to raise_error /No array columns were given/ } end + + context 'when Arel AS node is given as input' do + let(:scope) { Issue.select(Issue.arel_table[:id].as('id'), :title) } + let(:columns) { scope.select_values } + + it 'works with Arel AS nodes' do + expect(array_scope_columns.array_aggregated_column_names).to eq(%w[array_cte_id_array array_cte_title_array]) + end + end end diff --git a/spec/lib/gitlab/rack_attack/request_spec.rb b/spec/lib/gitlab/rack_attack/request_spec.rb index 9b882f26480..4e304de3224 100644 --- a/spec/lib/gitlab/rack_attack/request_spec.rb +++ b/spec/lib/gitlab/rack_attack/request_spec.rb @@ -12,7 +12,7 @@ RSpec.describe Gitlab::RackAttack::Request do ::Rack::Attack::Request.new( env.reverse_merge( 'REQUEST_METHOD' => 'GET', - 'PATH_INFO' => path, + 'PATH_INFO' => Gitlab.config.gitlab.relative_url_root + path, 'rack.input' => StringIO.new, 'rack.session' => session ) @@ -44,6 +44,14 @@ RSpec.describe Gitlab::RackAttack::Request do with_them do it { is_expected.to eq(expected) } + + context 'when the application is mounted at a relative URL' do + before do + stub_config_setting(relative_url_root: '/gitlab/root') + end + + it { is_expected.to eq(expected) } + end end end @@ -65,6 +73,14 @@ RSpec.describe Gitlab::RackAttack::Request do with_them do it { is_expected.to eq(expected) } + + context 'when the application is mounted at a relative URL' do + before do + stub_config_setting(relative_url_root: '/gitlab/root') + end + + it { is_expected.to eq(expected) } + end end end @@ -88,6 +104,14 @@ RSpec.describe Gitlab::RackAttack::Request do with_them do it { is_expected.to eq(expected) } + + context 'when the application is mounted at a relative URL' do + before do + stub_config_setting(relative_url_root: '/gitlab/root') + end + + it { is_expected.to eq(expected) } + end end end @@ -107,6 +131,14 @@ RSpec.describe Gitlab::RackAttack::Request do with_them do it { is_expected.to eq(expected) } + + context 'when the application is mounted at a relative URL' do + before do + stub_config_setting(relative_url_root: '/gitlab/root') + end + + it { is_expected.to eq(expected) } + end end end @@ -127,6 +159,14 @@ RSpec.describe Gitlab::RackAttack::Request do with_them do it { is_expected.to eq(expected) } + + context 'when the application is mounted at a relative URL' do + before do + stub_config_setting(relative_url_root: '/gitlab/root') + end + + it { is_expected.to eq(expected) } + end end end @@ -162,6 +202,14 @@ RSpec.describe Gitlab::RackAttack::Request do with_them do it { is_expected.to eq(expected) } + + context 'when the application is mounted at a relative URL' do + before do + stub_config_setting(relative_url_root: '/gitlab/root') + end + + it { is_expected.to eq(expected) } + end end end @@ -189,6 +237,14 @@ RSpec.describe Gitlab::RackAttack::Request do with_them do it { is_expected.to eq(expected) } + + context 'when the application is mounted at a relative URL' do + before do + stub_config_setting(relative_url_root: '/gitlab/root') + end + + it { is_expected.to eq(expected) } + end end end @@ -255,6 +311,14 @@ RSpec.describe Gitlab::RackAttack::Request do with_them do it { is_expected.to eq(expected) } + + context 'when the application is mounted at a relative URL' do + before do + stub_config_setting(relative_url_root: '/gitlab/root') + end + + it { is_expected.to eq(expected) } + end end end end diff --git a/spec/models/loose_foreign_keys/deleted_record_spec.rb b/spec/models/loose_foreign_keys/deleted_record_spec.rb index 07ffff746a5..23e0ed1f39d 100644 --- a/spec/models/loose_foreign_keys/deleted_record_spec.rb +++ b/spec/models/loose_foreign_keys/deleted_record_spec.rb @@ -6,15 +6,15 @@ RSpec.describe LooseForeignKeys::DeletedRecord, type: :model do let_it_be(:table) { 'public.projects' } describe 'class methods' do - let_it_be(:deleted_record_1) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 5) } - let_it_be(:deleted_record_2) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1) } + let_it_be(:deleted_record_1) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 5, cleanup_attempts: 2) } + let_it_be(:deleted_record_2) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1, cleanup_attempts: 0) } let_it_be(:deleted_record_3) { described_class.create!(fully_qualified_table_name: 'public.other_table', primary_key_value: 3) } - let_it_be(:deleted_record_4) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1) } # duplicate + let_it_be(:deleted_record_4) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1, cleanup_attempts: 1) } # duplicate + + let(:records) { described_class.load_batch_for_table(table, 10) } describe '.load_batch_for_table' do it 'loads records and orders them by creation date' do - records = described_class.load_batch_for_table(table, 10) - expect(records).to eq([deleted_record_1, deleted_record_2, deleted_record_4]) end @@ -27,13 +27,38 @@ RSpec.describe LooseForeignKeys::DeletedRecord, type: :model do describe '.mark_records_processed' do it 'updates all records' do - records = described_class.load_batch_for_table(table, 10) described_class.mark_records_processed(records) expect(described_class.status_pending.count).to eq(1) expect(described_class.status_processed.count).to eq(3) end end + + describe '.reschedule' do + it 'reschedules all records' do + time = Time.zone.parse('2022-01-01').utc + update_count = described_class.reschedule(records, time) + + expect(update_count).to eq(records.size) + + records.each(&:reload) + + expect(records).to all(have_attributes( + cleanup_attempts: 0, + consume_after: time + )) + end + end + + describe '.increment_attempts' do + it 'increaments the cleanup_attempts column' do + described_class.increment_attempts(records) + + expect(deleted_record_1.reload.cleanup_attempts).to eq(3) + expect(deleted_record_2.reload.cleanup_attempts).to eq(1) + expect(deleted_record_4.reload.cleanup_attempts).to eq(2) + end + end end describe 'sliding_list partitioning' do diff --git a/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb b/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb index d3d57ea2444..538d9638879 100644 --- a/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb +++ b/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb @@ -115,4 +115,82 @@ RSpec.describe LooseForeignKeys::BatchCleanerService do expect(loose_fk_child_table_2.where(parent_id_with_different_column: other_parent_record.id).count).to eq(2) end end + + describe 'fair queueing' do + context 'when the execution is over the limit' do + let(:modification_tracker) { instance_double(LooseForeignKeys::ModificationTracker) } + let(:over_limit_return_values) { [true] } + let(:deleted_record) { LooseForeignKeys::DeletedRecord.load_batch_for_table('public._test_loose_fk_parent_table', 1).first } + let(:deleted_records_rescheduled_counter) { Gitlab::Metrics.registry.get(:loose_foreign_key_rescheduled_deleted_records) } + let(:deleted_records_incremented_counter) { Gitlab::Metrics.registry.get(:loose_foreign_key_incremented_deleted_records) } + + let(:cleaner) do + described_class.new(parent_table: '_test_loose_fk_parent_table', + loose_foreign_key_definitions: loose_foreign_key_definitions, + deleted_parent_records: LooseForeignKeys::DeletedRecord.load_batch_for_table('public._test_loose_fk_parent_table', 100), + modification_tracker: modification_tracker + ) + end + + before do + parent_record_1.delete + allow(modification_tracker).to receive(:over_limit?).and_return(*over_limit_return_values) + allow(modification_tracker).to receive(:add_deletions) + end + + context 'when the deleted record is under the maximum allowed cleanup attempts' do + it 'updates the cleanup_attempts column', :aggregate_failures do + deleted_record.update!(cleanup_attempts: 1) + + cleaner.execute + + expect(deleted_record.reload.cleanup_attempts).to eq(2) + expect(deleted_records_incremented_counter.get(table: loose_fk_parent_table.table_name, db_config_name: 'main')).to eq(1) + end + + context 'when the deleted record is above the maximum allowed cleanup attempts' do + it 'reschedules the record', :aggregate_failures do + deleted_record.update!(cleanup_attempts: LooseForeignKeys::BatchCleanerService::CLEANUP_ATTEMPTS_BEFORE_RESCHEDULE + 1) + + freeze_time do + cleaner.execute + + expect(deleted_record.reload).to have_attributes( + cleanup_attempts: 0, + consume_after: 5.minutes.from_now + ) + expect(deleted_records_rescheduled_counter.get(table: loose_fk_parent_table.table_name, db_config_name: 'main')).to eq(1) + end + end + end + + describe 'when over limit happens on the second cleanup call without skip locked' do + # over_limit? is called twice, we test here the 2nd call + # - When invoking cleanup with SKIP LOCKED + # - When invoking cleanup (no SKIP LOCKED) + let(:over_limit_return_values) { [false, true] } + + it 'updates the cleanup_attempts column' do + expect(cleaner).to receive(:run_cleaner_service).twice + + deleted_record.update!(cleanup_attempts: 1) + + cleaner.execute + + expect(deleted_record.reload.cleanup_attempts).to eq(2) + end + end + end + + context 'when the lfk_fair_queueing FF is off' do + before do + stub_feature_flags(lfk_fair_queueing: false) + end + + it 'does nothing' do + expect { cleaner.execute }.not_to change { deleted_record.reload.cleanup_attempts } + end + end + end + end end |
