summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2022-02-07 09:12:13 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2022-02-07 09:12:13 +0000
commita7760cee245c8cffc1b322c6031023985e764a12 (patch)
treef99e6b6fa4a4e56a1142d8e4e2e2ca1ea5054df0
parent4ea8f39241d94ac4e42bcd44f7b2db80766d2472 (diff)
downloadgitlab-ce-a7760cee245c8cffc1b322c6031023985e764a12.tar.gz
Add latest changes from gitlab-org/gitlab@master
-rw-r--r--app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue1
-rw-r--r--app/assets/stylesheets/page_bundles/merge_requests.scss48
-rw-r--r--app/models/loose_foreign_keys/deleted_record.rb32
-rw-r--r--app/models/namespaces/traversal/linear_scopes.rb2
-rw-r--r--app/services/loose_foreign_keys/batch_cleaner_service.rb52
-rw-r--r--app/views/projects/settings/ci_cd/_form.html.haml2
-rw-r--r--config/feature_flags/development/lfk_fair_queueing.yml (renamed from config/feature_flags/development/deployment_approvals.yml)10
-rw-r--r--data/deprecations/14-9-deprecate-testcoveragesetting.yml14
-rw-r--r--db/migrate/20220125084127_add_cleanup_attempts_to_loose_foreign_keys_deleted_records.rb13
-rw-r--r--db/schema_migrations/202201250841271
-rw-r--r--db/structure.sql1
-rw-r--r--doc/api/deployments.md5
-rw-r--r--doc/ci/environments/deployment_approvals.md3
-rw-r--r--doc/ci/pipelines/settings.md10
-rw-r--r--doc/ci/yaml/index.md2
-rw-r--r--doc/update/deprecations.md17
-rw-r--r--doc/user/project/merge_requests/test_coverage_visualization.md2
-rw-r--r--lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb1
-rw-r--r--lib/gitlab/rack_attack/request.rb28
-rw-r--r--locale/gitlab.pot6
-rw-r--r--qa/Gemfile1
-rw-r--r--qa/qa/page/component/members_filter.rb26
-rw-r--r--qa/qa/page/group/members.rb3
-rw-r--r--qa/qa/page/project/members.rb1
-rw-r--r--qa/qa/resource/api_fabricator.rb17
-rw-r--r--qa/qa/resource/base.rb5
-rw-r--r--qa/qa/resource/clusters/agent.rb1
-rw-r--r--qa/qa/resource/clusters/agent_token.rb1
-rw-r--r--qa/qa/resource/reusable.rb4
-rw-r--r--qa/qa/runtime/env.rb4
-rw-r--r--qa/qa/runtime/logger.rb10
-rw-r--r--qa/qa/service/cluster_provider/gcloud.rb2
-rw-r--r--qa/qa/service/shellout.rb3
-rw-r--r--qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb70
-rw-r--r--qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb8
-rw-r--r--qa/qa/support/page/logging.rb12
-rw-r--r--qa/qa/support/repeater.rb5
-rw-r--r--qa/qa/tools/initialize_gitlab_auth.rb2
-rw-r--r--qa/spec/spec_helper.rb1
-rw-r--r--spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb9
-rw-r--r--spec/lib/gitlab/rack_attack/request_spec.rb66
-rw-r--r--spec/models/loose_foreign_keys/deleted_record_spec.rb37
-rw-r--r--spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb78
43 files changed, 492 insertions, 124 deletions
diff --git a/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue b/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue
index e9329fb1d88..633dee75237 100644
--- a/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue
+++ b/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue
@@ -151,6 +151,7 @@ export default {
:search-input-placeholder="filteredSearchBar.placeholder"
:initial-filter-value="initialFilterValue"
data-testid="members-filtered-search-bar"
+ data-qa-selector="members_filtered_search_bar_content"
@onFilter="handleFilter"
/>
</template>
diff --git a/app/assets/stylesheets/page_bundles/merge_requests.scss b/app/assets/stylesheets/page_bundles/merge_requests.scss
index 1ee154f9aeb..6c9c648a4b0 100644
--- a/app/assets/stylesheets/page_bundles/merge_requests.scss
+++ b/app/assets/stylesheets/page_bundles/merge_requests.scss
@@ -132,7 +132,7 @@ $tabs-holder-z-index: 250;
.coverage {
font-size: 12px;
- color: $gray-500;
+ color: var(--gray-500, $gray-500);
line-height: initial;
}
}
@@ -194,12 +194,12 @@ $tabs-holder-z-index: 250;
}
&:not(:last-child) {
- border-bottom: 1px solid $border-color;
+ border-bottom: 1px solid var(--border-color, $border-color);
}
}
.diff-file-row.is-active {
- background-color: $gray-50;
+ background-color: var(--gray-50, $gray-50);
}
.mr-conflict-loader {
@@ -255,7 +255,7 @@ $tabs-holder-z-index: 250;
}
.mr-section-container {
- border: 1px solid $border-color;
+ border: 1px solid var(--border-color, $border-color);
border-radius: $border-radius-default;
background: var(--white, $white);
@@ -299,19 +299,19 @@ $tabs-holder-z-index: 250;
}
.diverged-commits-count {
- color: $gl-text-color-secondary;
+ color: var(--gray-500, $gl-text-color-secondary);
}
}
.mr-state-widget {
- color: $gl-text-color;
+ color: var(--gl-text-color, $gl-text-color);
.commit-message-edit {
border-radius: $border-radius-default;
}
.mr-widget-section:not(:first-child) {
- border-top: solid 1px $border-color;
+ border-top: solid 1px var(--border-color, $border-color);
}
.mr-widget-alert-container + .mr-widget-section {
@@ -373,7 +373,7 @@ $tabs-holder-z-index: 250;
}
.ci-widget {
- color: $gl-text-color;
+ color: var(--gl-text-color, $gl-text-color);
display: flex;
align-items: center;
justify-content: space-between;
@@ -422,7 +422,7 @@ $tabs-holder-z-index: 250;
.label-branch {
@include gl-font-monospace;
font-size: 95%;
- color: $gl-text-color;
+ color: var(--gl-text-color, $gl-text-color);
font-weight: normal;
overflow: hidden;
word-break: break-all;
@@ -486,7 +486,7 @@ $tabs-holder-z-index: 250;
.bold,
.gl-font-weight-bold {
font-weight: $gl-font-weight-bold;
- color: $gray-600;
+ color: var(--gray-600, $gray-600);
margin-left: 10px;
}
@@ -496,7 +496,7 @@ $tabs-holder-z-index: 250;
}
.danger {
- color: $red-500;
+ color: var(--red-500, $red-500);
}
.spacing,
@@ -571,10 +571,10 @@ $tabs-holder-z-index: 250;
&.mr-pipeline-suggest {
border-radius: $border-radius-default;
line-height: 20px;
- border: 1px solid $border-color;
+ border: 1px solid var(--border-color, $border-color);
.circle-icon-container {
- color: $gl-text-color-quaternary;
+ color: var(--gray-100, $gl-text-color-quaternary);
}
}
}
@@ -584,11 +584,11 @@ $tabs-holder-z-index: 250;
}
.stop-env-container {
- color: $gl-text-color;
+ color: var(--gl-text-color, $gl-text-color);
float: right;
a {
- color: $gl-text-color;
+ color: var(--gl-text-color, $gl-text-color);
}
}
}
@@ -609,15 +609,15 @@ $tabs-holder-z-index: 250;
}
.mr-widget-border-top {
- border-top: 1px solid $border-color;
+ border-top: 1px solid var(--border-color, $border-color);
}
.mr-widget-extension {
- border-top: 1px solid $border-color;
- background-color: $gray-50;
+ border-top: 1px solid var(--border-color, $border-color);
+ background-color: var(--gray-50, $gray-50);
&.clickable:hover {
- background-color: $gray-100;
+ background-color: var(--gray-100, $gray-100);
cursor: pointer;
}
}
@@ -638,7 +638,7 @@ $tabs-holder-z-index: 250;
.mr-widget-heading {
position: relative;
- border: 1px solid $border-color;
+ border: 1px solid var(--border-color, $border-color);
border-radius: $border-radius-default;
background: var(--white, $white);
@@ -668,7 +668,7 @@ $tabs-holder-z-index: 250;
&::before {
content: '';
- border-left: 1px solid $gray-100;
+ border-left: 1px solid var(--gray-100, $gray-100);
position: absolute;
left: 28px;
top: -17px;
@@ -679,8 +679,8 @@ $tabs-holder-z-index: 250;
.mr-version-controls {
position: relative;
z-index: $tabs-holder-z-index + 10;
- background: $white;
- color: $gl-text-color;
+ background: var(--white, $white);
+ color: var(--gl-text-color, $gl-text-color);
margin-top: -1px;
.mr-version-menus-container {
@@ -713,7 +713,7 @@ $tabs-holder-z-index: 250;
}
.dropdown-title {
- color: $gl-text-color;
+ color: var(--gl-text-color, $gl-text-color);
}
// Shortening button height by 1px to make compare-versions
diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb
index db82d5bbf29..ebda5872f1c 100644
--- a/app/models/loose_foreign_keys/deleted_record.rb
+++ b/app/models/loose_foreign_keys/deleted_record.rb
@@ -46,17 +46,39 @@ class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel
.to_a
end
- def self.mark_records_processed(all_records)
- # Run a query for each partition to optimize the row lookup by primary key (partition, id)
+ def self.mark_records_processed(records)
+ update_by_partition(records) do |partitioned_scope|
+ partitioned_scope.update_all(status: :processed)
+ end
+ end
+
+ def self.reschedule(records, consume_after)
+ update_by_partition(records) do |partitioned_scope|
+ partitioned_scope.update_all(consume_after: consume_after, cleanup_attempts: 0)
+ end
+ end
+
+ def self.increment_attempts(records)
+ update_by_partition(records) do |partitioned_scope|
+ # Naive incrementing of the cleanup_attempts is good enough for us.
+ partitioned_scope.update_all('cleanup_attempts = cleanup_attempts + 1')
+ end
+ end
+
+ def self.update_by_partition(records)
update_count = 0
- all_records.group_by(&:partition_number).each do |partition, records_within_partition|
- update_count += status_pending
+ # Run a query for each partition to optimize the row lookup by primary key (partition, id)
+ records.group_by(&:partition_number).each do |partition, records_within_partition|
+ partitioned_scope = status_pending
.for_partition(partition)
.where(id: records_within_partition.pluck(:id))
- .update_all(status: :processed)
+
+ update_count += yield(partitioned_scope)
end
update_count
end
+
+ private_class_method :update_by_partition
end
diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb
index cfc993efd9a..fed86c020dd 100644
--- a/app/models/namespaces/traversal/linear_scopes.rb
+++ b/app/models/namespaces/traversal/linear_scopes.rb
@@ -12,7 +12,7 @@ module Namespaces
def as_ids
return super unless use_traversal_ids?
- select('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id')
+ select(Arel.sql('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)]').as('id'))
end
def roots
diff --git a/app/services/loose_foreign_keys/batch_cleaner_service.rb b/app/services/loose_foreign_keys/batch_cleaner_service.rb
index de52cbba576..f3db2037911 100644
--- a/app/services/loose_foreign_keys/batch_cleaner_service.rb
+++ b/app/services/loose_foreign_keys/batch_cleaner_service.rb
@@ -2,6 +2,9 @@
module LooseForeignKeys
class BatchCleanerService
+ CLEANUP_ATTEMPTS_BEFORE_RESCHEDULE = 3
+ CONSUME_AFTER_RESCHEDULE = 5.minutes
+
def initialize(parent_table:, loose_foreign_key_definitions:, deleted_parent_records:, modification_tracker: LooseForeignKeys::ModificationTracker.new)
@parent_table = parent_table
@loose_foreign_key_definitions = loose_foreign_key_definitions
@@ -11,15 +14,31 @@ module LooseForeignKeys
:loose_foreign_key_processed_deleted_records,
'The number of processed loose foreign key deleted records'
)
+ @deleted_records_rescheduled_count = Gitlab::Metrics.counter(
+ :loose_foreign_key_rescheduled_deleted_records,
+ 'The number of rescheduled loose foreign key deleted records'
+ )
+ @deleted_records_incremented_count = Gitlab::Metrics.counter(
+ :loose_foreign_key_incremented_deleted_records,
+ 'The number of loose foreign key deleted records with incremented cleanup_attempts'
+ )
end
def execute
loose_foreign_key_definitions.each do |loose_foreign_key_definition|
run_cleaner_service(loose_foreign_key_definition, with_skip_locked: true)
- break if modification_tracker.over_limit?
+
+ if modification_tracker.over_limit?
+ handle_over_limit
+ break
+ end
run_cleaner_service(loose_foreign_key_definition, with_skip_locked: false)
- break if modification_tracker.over_limit?
+
+ if modification_tracker.over_limit?
+ handle_over_limit
+ break
+ end
end
return if modification_tracker.over_limit?
@@ -27,12 +46,33 @@ module LooseForeignKeys
# At this point, all associations are cleaned up, we can update the status of the parent records
update_count = LooseForeignKeys::DeletedRecord.mark_records_processed(deleted_parent_records)
- deleted_records_counter.increment({ table: parent_table, db_config_name: LooseForeignKeys::DeletedRecord.connection.pool.db_config.name }, update_count)
+ deleted_records_counter.increment({ table: parent_table, db_config_name: db_config_name }, update_count)
end
private
- attr_reader :parent_table, :loose_foreign_key_definitions, :deleted_parent_records, :modification_tracker, :deleted_records_counter
+ attr_reader :parent_table, :loose_foreign_key_definitions, :deleted_parent_records, :modification_tracker, :deleted_records_counter, :deleted_records_rescheduled_count, :deleted_records_incremented_count
+
+ def handle_over_limit
+ return if Feature.disabled?(:lfk_fair_queueing)
+
+ records_to_reschedule = []
+ records_to_increment = []
+
+ deleted_parent_records.each do |deleted_record|
+ if deleted_record.cleanup_attempts >= CLEANUP_ATTEMPTS_BEFORE_RESCHEDULE
+ records_to_reschedule << deleted_record
+ else
+ records_to_increment << deleted_record
+ end
+ end
+
+ reschedule_count = LooseForeignKeys::DeletedRecord.reschedule(records_to_reschedule, CONSUME_AFTER_RESCHEDULE.from_now)
+ deleted_records_rescheduled_count.increment({ table: parent_table, db_config_name: db_config_name }, reschedule_count)
+
+ increment_count = LooseForeignKeys::DeletedRecord.increment_attempts(records_to_increment)
+ deleted_records_incremented_count.increment({ table: parent_table, db_config_name: db_config_name }, increment_count)
+ end
def record_result(cleaner, result)
if cleaner.async_delete?
@@ -60,5 +100,9 @@ module LooseForeignKeys
end
end
end
+
+ def db_config_name
+ LooseForeignKeys::DeletedRecord.connection.pool.db_config.name
+ end
end
end
diff --git a/app/views/projects/settings/ci_cd/_form.html.haml b/app/views/projects/settings/ci_cd/_form.html.haml
index d3cdfc4f7c9..c70e153ae41 100644
--- a/app/views/projects/settings/ci_cd/_form.html.haml
+++ b/app/views/projects/settings/ci_cd/_form.html.haml
@@ -94,7 +94,7 @@
.input-group-text /
%p.form-text.text-muted
= html_escape(_('The regular expression used to find test coverage output in the job log. For example, use %{regex} for Simplecov (Ruby). Leave blank to disable.')) % { regex: '<code>\(\d+.\d+%\)</code>'.html_safe }
- = link_to sprite_icon('question-o'), help_page_path('ci/pipelines/settings', anchor: 'add-test-coverage-results-to-a-merge-request'), target: '_blank', rel: 'noopener noreferrer'
+ = link_to sprite_icon('question-o'), help_page_path('ci/pipelines/settings', anchor: 'add-test-coverage-results-to-a-merge-request-deprecated'), target: '_blank', rel: 'noopener noreferrer'
= f.submit _('Save changes'), class: "btn gl-button btn-confirm", data: { qa_selector: 'save_general_pipelines_changes_button' }
diff --git a/config/feature_flags/development/deployment_approvals.yml b/config/feature_flags/development/lfk_fair_queueing.yml
index 5083ccd28bf..ac67ffa14f0 100644
--- a/config/feature_flags/development/deployment_approvals.yml
+++ b/config/feature_flags/development/lfk_fair_queueing.yml
@@ -1,8 +1,8 @@
---
-name: deployment_approvals
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74932
-rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/347342
-milestone: '14.6'
+name: lfk_fair_queueing
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79116
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/351082
+milestone: '14.8'
type: development
-group: group::release
+group: group::sharding
default_enabled: false
diff --git a/data/deprecations/14-9-deprecate-testcoveragesetting.yml b/data/deprecations/14-9-deprecate-testcoveragesetting.yml
new file mode 100644
index 00000000000..19bc6004b02
--- /dev/null
+++ b/data/deprecations/14-9-deprecate-testcoveragesetting.yml
@@ -0,0 +1,14 @@
+- name: "Test coverage project CI/CD setting" # The name of the feature to be deprecated
+ announcement_milestone: "14.8" # The milestone when this feature was first announced as deprecated.
+ announcement_date: "2022-02-22" # The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post.
+ removal_milestone: "15.0" # The milestone when this feature is planned to be removed
+ removal_date: "2022-05-22" # The date of the milestone release when this feature is planned to be removed. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post.
+ breaking_change: true # If this deprecation is a breaking change, set this value to true
+ reporter: jreporter # GitLab username of the person reporting the deprecation
+ body: | # Do not modify this line, instead modify the lines below.
+ To simplify setting a test coverage pattern, in GitLab 15.0 the
+ [project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-to-a-merge-request-deprecated)
+ is being removed.
+
+ Instead, using the project’s `.gitlab-ci.yml`, provide a regular expression with the `coverage` keyword to set
+ testing coverage results in merge requests.
diff --git a/db/migrate/20220125084127_add_cleanup_attempts_to_loose_foreign_keys_deleted_records.rb b/db/migrate/20220125084127_add_cleanup_attempts_to_loose_foreign_keys_deleted_records.rb
new file mode 100644
index 00000000000..e0c80ad79e5
--- /dev/null
+++ b/db/migrate/20220125084127_add_cleanup_attempts_to_loose_foreign_keys_deleted_records.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddCleanupAttemptsToLooseForeignKeysDeletedRecords < Gitlab::Database::Migration[1.0]
+ enable_lock_retries!
+
+ def up
+ add_column :loose_foreign_keys_deleted_records, :cleanup_attempts, :smallint, default: 0
+ end
+
+ def down
+ remove_column :loose_foreign_keys_deleted_records, :cleanup_attempts
+ end
+end
diff --git a/db/schema_migrations/20220125084127 b/db/schema_migrations/20220125084127
new file mode 100644
index 00000000000..f9ba7a4628b
--- /dev/null
+++ b/db/schema_migrations/20220125084127
@@ -0,0 +1 @@
+c50a21430e52fc6ccbe7ab4aba99ae3a48d1c69858f7886da115f54e19fc27ea \ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index dad5942f213..6dc825fdedb 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -279,6 +279,7 @@ CREATE TABLE loose_foreign_keys_deleted_records (
created_at timestamp with time zone DEFAULT now() NOT NULL,
fully_qualified_table_name text NOT NULL,
consume_after timestamp with time zone DEFAULT now(),
+ cleanup_attempts smallint DEFAULT 0,
CONSTRAINT check_1a541f3235 CHECK ((char_length(fully_qualified_table_name) <= 150))
)
PARTITION BY LIST (partition);
diff --git a/doc/api/deployments.md b/doc/api/deployments.md
index 7fc870006ac..29f9bd3e2ee 100644
--- a/doc/api/deployments.md
+++ b/doc/api/deployments.md
@@ -448,7 +448,10 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a
## Approve or reject a blocked deployment **(PREMIUM)**
-> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343864) in GitLab 14.7 [with a flag](../administration/feature_flags.md) named `deployment_approvals`. Disabled by default. This feature is not ready for production use.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343864) in GitLab 14.7 [with a flag](../administration/feature_flags.md) named `deployment_approvals`. Disabled by default.
+> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/347342) in GitLab 14.8.
+
+See [Deployment Approvals](../ci/environments/deployment_approvals.md) for more information about this feature.
```plaintext
POST /projects/:id/deployments/:deployment_id/approval
diff --git a/doc/ci/environments/deployment_approvals.md b/doc/ci/environments/deployment_approvals.md
index a82da42d24a..d4550f1da97 100644
--- a/doc/ci/environments/deployment_approvals.md
+++ b/doc/ci/environments/deployment_approvals.md
@@ -7,7 +7,8 @@ description: Require approvals prior to deploying to a Protected Environment
# Deployment approvals **(PREMIUM)**
-> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343864) in GitLab 14.7 with a flag named `deployment_approvals`. Disabled by default.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343864) in GitLab 14.7 with a flag named `deployment_approvals`. Disabled by default.
+> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/347342) in GitLab 14.8.
WARNING:
This feature is in an alpha stage and subject to change without prior notice.
diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md
index 7db57fc44ff..e22746dbfa0 100644
--- a/doc/ci/pipelines/settings.md
+++ b/doc/ci/pipelines/settings.md
@@ -195,7 +195,13 @@ Jobs that exceed the timeout are marked as failed.
You can override this value [for individual runners](../runners/configure_runners.md#set-maximum-job-timeout-for-a-runner).
-## Add test coverage results to a merge request
+## Add test coverage results to a merge request (DEPRECATED)
+
+> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 14.9. Replaced by [`coverage` keyword](../yaml/index.md#coverage).
+
+WARNING:
+This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633)
+for use in GitLab 14.9, and is planned for [removal](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 15.0.
If you use test coverage in your code, you can use a regular expression to
find coverage results in the job log. You can then include these results
@@ -358,7 +364,7 @@ https://gitlab.example.com/<namespace>/<project>/badges/<branch>/pipeline.svg?ig
### Test coverage report badge
-You can define the regular expression for the [coverage report](#add-test-coverage-results-to-a-merge-request)
+You can define the regular expression for the [coverage report](#add-test-coverage-results-to-a-merge-request-deprecated)
that each job log is matched against. This means that each job in the
pipeline can have the test coverage percentage value defined.
diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md
index c4c897571e7..99e4536e94f 100644
--- a/doc/ci/yaml/index.md
+++ b/doc/ci/yaml/index.md
@@ -1349,7 +1349,7 @@ In this example:
**Additional details**:
- Coverage regular expressions set in `gitlab-ci.yml` take precedence over coverage regular expression set in the
- [GitLab UI](../pipelines/settings.md#add-test-coverage-results-to-a-merge-request).
+ [GitLab UI](../pipelines/settings.md#add-test-coverage-results-to-a-merge-request-deprecated).
- If there is more than one matched line in the job output, the last line is used
(the first result of reverse search).
- If there are multiple matches in a single line, the last match is searched
diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md
index 1e4d13a0039..77a6311f3c2 100644
--- a/doc/update/deprecations.md
+++ b/doc/update/deprecations.md
@@ -853,6 +853,23 @@ existing runners.
**Planned removal milestone: 15.0 (2022-05-22)**
+### Test coverage project CI/CD setting
+
+WARNING:
+This feature will be changed or removed in 15.0
+as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes).
+Before updating GitLab, review the details carefully to determine if you need to make any
+changes to your code, settings, or workflow.
+
+To simplify setting a test coverage pattern, in GitLab 15.0 the
+[project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-to-a-merge-request-deprecated)
+is being removed.
+
+Instead, using the project’s `.gitlab-ci.yml`, provide a regular expression with the `coverage` keyword to set
+testing coverage results in merge requests.
+
+**Planned removal milestone: 15.0 (2022-05-22)**
+
### Vulnerability Check
WARNING:
diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md
index c67bd3de8a6..728d28d293c 100644
--- a/doc/user/project/merge_requests/test_coverage_visualization.md
+++ b/doc/user/project/merge_requests/test_coverage_visualization.md
@@ -53,7 +53,7 @@ from any job in any stage in the pipeline. The coverage displays for each line:
Hovering over the coverage bar provides further information, such as the number
of times the line was checked by tests.
-Uploading a test coverage report does not enable [test coverage results in Merge Requests](../../../ci/pipelines/settings.md#add-test-coverage-results-to-a-merge-request) or [code coverage history](../../../ci/pipelines/settings.md#view-code-coverage-history). You must configure these separately.
+Uploading a test coverage report does not enable [test coverage results in Merge Requests](../../../ci/pipelines/settings.md#add-test-coverage-results-to-a-merge-request-deprecated) or [code coverage history](../../../ci/pipelines/settings.md#view-code-coverage-history). You must configure these separately.
### Limits
diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb
index 95afd5a8595..58963779c69 100644
--- a/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb
+++ b/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns.rb
@@ -12,6 +12,7 @@ module Gitlab
array_scope_table = Arel::Table.new(ARRAY_SCOPE_CTE_NAME)
@columns = columns.map do |column|
+ column = column.right if column.is_a?(Arel::Nodes::As)
ColumnData.new(column, "array_scope_#{column}", array_scope_table)
end
end
diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb
index 1e50df5f21c..53a0badfc30 100644
--- a/lib/gitlab/rack_attack/request.rb
+++ b/lib/gitlab/rack_attack/request.rb
@@ -28,23 +28,31 @@ module Gitlab
end
def api_request?
- path.start_with?('/api')
+ logical_path.start_with?('/api')
+ end
+
+ def logical_path
+ @logical_path ||= path.delete_prefix(Gitlab.config.gitlab.relative_url_root)
+ end
+
+ def matches?(regex)
+ logical_path.match?(regex)
end
def api_internal_request?
- path.match?(%r{^/api/v\d+/internal/})
+ matches?(%r{^/api/v\d+/internal/})
end
def health_check_request?
- path.match?(%r{^/-/(health|liveness|readiness|metrics)})
+ matches?(%r{^/-/(health|liveness|readiness|metrics)})
end
def container_registry_event?
- path.match?(%r{^/api/v\d+/container_registry_event/})
+ matches?(%r{^/api/v\d+/container_registry_event/})
end
def product_analytics_collector_request?
- path.start_with?('/-/collector/i')
+ logical_path.start_with?('/-/collector/i')
end
def should_be_skipped?
@@ -56,7 +64,7 @@ module Gitlab
end
def protected_path?
- path.match?(protected_paths_regex)
+ matches?(protected_paths_regex)
end
def throttle?(throttle, authenticated:)
@@ -178,15 +186,15 @@ module Gitlab
end
def packages_api_path?
- path.match?(::Gitlab::Regex::Packages::API_PATH_REGEX)
+ matches?(::Gitlab::Regex::Packages::API_PATH_REGEX)
end
def git_lfs_path?
- path.match?(::Gitlab::PathRegex.repository_git_lfs_route_regex)
+ matches?(::Gitlab::PathRegex.repository_git_lfs_route_regex)
end
def files_api_path?
- path.match?(FILES_PATH_REGEX)
+ matches?(FILES_PATH_REGEX)
end
def frontend_request?
@@ -206,7 +214,7 @@ module Gitlab
with_projects = params['with_projects']
with_projects = true if with_projects.blank?
- path.match?(GROUP_PATH_REGEX) && Gitlab::Utils.to_boolean(with_projects)
+ matches?(GROUP_PATH_REGEX) && Gitlab::Utils.to_boolean(with_projects)
end
end
end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 46c5e766830..279a76bbfdb 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -31974,9 +31974,6 @@ msgstr ""
msgid "SecurityOrchestration|Description"
msgstr ""
-msgid "SecurityOrchestration|Disabled"
-msgstr ""
-
msgid "SecurityOrchestration|Edit policy"
msgstr ""
@@ -32010,6 +32007,9 @@ msgstr ""
msgid "SecurityOrchestration|No rules defined - policy will not run."
msgstr ""
+msgid "SecurityOrchestration|Not enabled"
+msgstr ""
+
msgid "SecurityOrchestration|Only owners can update Security Policy Project"
msgstr ""
diff --git a/qa/Gemfile b/qa/Gemfile
index c07f9dc96a7..5373349d0e8 100644
--- a/qa/Gemfile
+++ b/qa/Gemfile
@@ -20,6 +20,7 @@ gem 'parallel_tests', '~> 2.29'
gem 'rotp', '~> 3.1.0'
gem 'timecop', '~> 0.9.1'
gem 'parallel', '~> 1.19'
+gem 'rainbow', '~> 3.0.0'
gem 'rspec-parameterized', '~> 0.4.2'
gem 'octokit', '~> 4.21'
gem 'webdrivers', '~> 5.0'
diff --git a/qa/qa/page/component/members_filter.rb b/qa/qa/page/component/members_filter.rb
new file mode 100644
index 00000000000..ac07fe7e9fa
--- /dev/null
+++ b/qa/qa/page/component/members_filter.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module QA
+ module Page
+ module Component
+ module MembersFilter
+ extend QA::Page::PageConcern
+
+ def self.included(base)
+ super
+
+ base.view 'app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue' do
+ element :members_filtered_search_bar_content
+ end
+ end
+
+ def search_member(username)
+ # TODO: Update the two actions below to use direct qa selectors once this is implemented:
+ # https://gitlab.com/gitlab-org/gitlab-ui/-/issues/1688
+ find_element(:members_filtered_search_bar_content).find('input').set(username)
+ find('.gl-search-box-by-click-search-button').click
+ end
+ end
+ end
+ end
+end
diff --git a/qa/qa/page/group/members.rb b/qa/qa/page/group/members.rb
index ccc901932f4..c80bdadb11f 100644
--- a/qa/qa/page/group/members.rb
+++ b/qa/qa/page/group/members.rb
@@ -6,6 +6,7 @@ module QA
class Members < Page::Base
include Page::Component::InviteMembersModal
include Page::Component::UsersSelect
+ include Page::Component::MembersFilter
view 'app/assets/javascripts/members/components/modals/remove_member_modal.vue' do
element :remove_member_modal_content
@@ -31,6 +32,8 @@ module QA
end
def update_access_level(username, access_level)
+ search_member(username)
+
within_element(:member_row, text: username) do
click_element :access_level_dropdown
click_element :access_level_link, text: access_level
diff --git a/qa/qa/page/project/members.rb b/qa/qa/page/project/members.rb
index 1102abd6646..30748ed920b 100644
--- a/qa/qa/page/project/members.rb
+++ b/qa/qa/page/project/members.rb
@@ -5,6 +5,7 @@ module QA
module Project
class Members < Page::Base
include QA::Page::Component::InviteMembersModal
+ include QA::Page::Component::MembersFilter
view 'app/assets/javascripts/members/components/members_tabs.vue' do
element :groups_list_tab
diff --git a/qa/qa/resource/api_fabricator.rb b/qa/qa/resource/api_fabricator.rb
index 4c77c515cfd..1958884916c 100644
--- a/qa/qa/resource/api_fabricator.rb
+++ b/qa/qa/resource/api_fabricator.rb
@@ -7,11 +7,11 @@ module QA
module Resource
module ApiFabricator
include Capybara::DSL
+ include Support::API
include Errors
- attr_reader :api_resource, :api_response
attr_writer :api_client
- attr_accessor :api_user
+ attr_accessor :api_user, :api_resource, :api_response
def api_support?
respond_to?(:api_get_path) &&
@@ -48,9 +48,6 @@ module QA
end
end
- include Support::API
- attr_writer :api_resource, :api_response
-
def api_put(body = api_put_body)
response = put(
Runtime::API::Request.new(api_client, api_put_path).url,
@@ -67,6 +64,16 @@ module QA
@api_fabrication_http_method ||= :post
end
+ # Checks if a resource already exists
+ #
+ # @return [Boolean] true if the resource returns HTTP status code 200
+ def exists?
+ request = Runtime::API::Request.new(api_client, api_get_path)
+ response = get(request.url)
+
+ response.code == HTTP_STATUS_OK
+ end
+
private
def resource_web_url(resource)
diff --git a/qa/qa/resource/base.rb b/qa/qa/resource/base.rb
index f8522872ddf..182a210d4ea 100644
--- a/qa/qa/resource/base.rb
+++ b/qa/qa/resource/base.rb
@@ -8,6 +8,7 @@ module QA
class Base
include ApiFabricator
extend Capybara::DSL
+ using Rainbow
NoValueError = Class.new(RuntimeError)
@@ -102,7 +103,7 @@ module QA
Runtime::Logger.debug do
msg = ["==#{'=' * parents.size}>"]
- msg << "#{fabrication_http_method} a #{name}"
+ msg << "#{fabrication_http_method} a #{Rainbow(name).black.bg(:white)}"
msg << identifier
msg << "as a dependency of #{parents.last}" if parents.any?
msg << "via #{fabrication_method}"
@@ -182,7 +183,7 @@ module QA
end
def visit!(skip_resp_code_check: false)
- Runtime::Logger.debug(%(Visiting #{self.class.name} at "#{web_url}"))
+ Runtime::Logger.debug("Visiting #{Rainbow(self.class.name).black.bg(:white)} at #{web_url}")
# Just in case an async action is not yet complete
Support::WaitForRequests.wait_for_requests(skip_resp_code_check: skip_resp_code_check)
diff --git a/qa/qa/resource/clusters/agent.rb b/qa/qa/resource/clusters/agent.rb
index ee5a292b9b3..b190634f357 100644
--- a/qa/qa/resource/clusters/agent.rb
+++ b/qa/qa/resource/clusters/agent.rb
@@ -17,7 +17,6 @@ module QA
end
def fabricate!
- puts 'TODO: FABRICATE VIA UI'
end
def resource_web_url(resource)
diff --git a/qa/qa/resource/clusters/agent_token.rb b/qa/qa/resource/clusters/agent_token.rb
index 6d803b94564..c1cf5c2f37b 100644
--- a/qa/qa/resource/clusters/agent_token.rb
+++ b/qa/qa/resource/clusters/agent_token.rb
@@ -11,7 +11,6 @@ module QA
end
def fabricate!
- puts 'TODO: FABRICATE VIA UI'
end
def resource_web_url(resource)
diff --git a/qa/qa/resource/reusable.rb b/qa/qa/resource/reusable.rb
index 24b0a1f6bce..02316a045c7 100644
--- a/qa/qa/resource/reusable.rb
+++ b/qa/qa/resource/reusable.rb
@@ -46,7 +46,9 @@ module QA
# @return [Hash<Symbol, QA::Resource>] the resources that were to be removed.
def remove_all_via_api!
resources.each do |reuse_as, resource|
- QA::Runtime::Logger.debug("#{self.name} - removing #{reuse_as}")
+ QA::Runtime::Logger.debug("#{self.name} - removing resource reused as :#{reuse_as}")
+ next QA::Runtime::Logger.debug("#{self.name} reused as :#{reuse_as} has already been removed.") unless resource.exists?
+
resource.method(:remove_via_api!).super_method.call
end
end
diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb
index cb4d1418145..3e442814734 100644
--- a/qa/qa/runtime/env.rb
+++ b/qa/qa/runtime/env.rb
@@ -65,6 +65,10 @@ module QA
ENV['QA_LOG_PATH'] || $stdout
end
+ def colorized_logs?
+ enabled?(ENV['COLORIZED_LOGS'], default: false)
+ end
+
# set to 'false' to have the browser run visibly instead of headless
def webdriver_headless?
if ENV.key?('CHROME_HEADLESS')
diff --git a/qa/qa/runtime/logger.rb b/qa/qa/runtime/logger.rb
index a70c8faf7d2..81c41000033 100644
--- a/qa/qa/runtime/logger.rb
+++ b/qa/qa/runtime/logger.rb
@@ -2,11 +2,13 @@
require 'logger'
require 'forwardable'
+require 'rainbow/refinement'
module QA
module Runtime
module Logger
extend SingleForwardable
+ using Rainbow
def_delegators :logger, :debug, :info, :warn, :error, :fatal, :unknown
@@ -14,8 +16,16 @@ module QA
attr_writer :logger
def logger
+ Rainbow.enabled = Runtime::Env.colorized_logs?
+
@logger ||= ::Logger.new(Runtime::Env.log_destination).tap do |logger|
logger.level = Runtime::Env.debug? ? ::Logger::DEBUG : ::Logger::ERROR
+
+ logger.formatter = proc do |severity, datetime, progname, msg|
+ date_format = datetime.strftime("%Y-%m-%d %H:%M:%S")
+
+ "[date=#{date_format} from=QA Tests] #{severity.ljust(5)} -- ".yellow + "#{msg}\n"
+ end
end
end
end
diff --git a/qa/qa/service/cluster_provider/gcloud.rb b/qa/qa/service/cluster_provider/gcloud.rb
index c6d1f6cfe88..77677745f7a 100644
--- a/qa/qa/service/cluster_provider/gcloud.rb
+++ b/qa/qa/service/cluster_provider/gcloud.rb
@@ -49,7 +49,7 @@ module QA
if account.empty?
raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally."
else
- puts "gcloud account found. Using: #{account} for creating K8s cluster."
+ QA::Runtime::Logger.debug("gcloud account found. Using: #{account} for creating K8s cluster.")
end
end
end
diff --git a/qa/qa/service/shellout.rb b/qa/qa/service/shellout.rb
index 33f2bc6e255..33d1d10b515 100644
--- a/qa/qa/service/shellout.rb
+++ b/qa/qa/service/shellout.rb
@@ -5,6 +5,7 @@ require 'open3'
module QA
module Service
module Shellout
+ using Rainbow
CommandError = Class.new(StandardError)
module_function
@@ -14,7 +15,7 @@ module QA
# as a library - gitlab-org/gitlab-qa#94
#
def shell(command, stdin_data: nil, fail_on_exception: true)
- puts "Executing `#{command}`"
+ QA::Runtime::Logger.info("Executing `#{command}`".cyan)
Open3.popen2e(*command) do |stdin, out, wait|
stdin.puts(stdin_data) if stdin_data
diff --git a/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb b/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb
index 6a31d173440..fe6c89f4ee4 100644
--- a/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb
+++ b/qa/qa/specs/features/api/1_manage/user_access_termination_spec.rb
@@ -31,44 +31,50 @@ module QA
end
it 'is not allowed to push code via the CLI', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347863' do
- expect do
- Resource::Repository::Push.fabricate! do |push|
- push.repository_http_uri = @project.repository_http_location.uri
- push.file_name = 'test.txt'
- push.file_content = "# This is a test project named #{@project.name}"
- push.commit_message = 'Add test.txt'
- push.branch_name = "new_branch_#{SecureRandom.hex(8)}"
- push.user = @user
- end
- end.to raise_error(QA::Support::Run::CommandError, /You are not allowed to push code to this project/)
+ QA::Support::Retrier.retry_on_exception(max_attempts: 5, sleep_interval: 2) do
+ expect do
+ Resource::Repository::Push.fabricate! do |push|
+ push.repository_http_uri = @project.repository_http_location.uri
+ push.file_name = 'test.txt'
+ push.file_content = "# This is a test project named #{@project.name}"
+ push.commit_message = 'Add test.txt'
+ push.branch_name = "new_branch_#{SecureRandom.hex(8)}"
+ push.user = @user
+ end
+ end.to raise_error(QA::Support::Run::CommandError, /You are not allowed to push code to this project/)
+ end
end
it 'is not allowed to create a file via the API', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347864' do
- expect do
- Resource::File.fabricate_via_api! do |file|
- file.api_client = @user_api_client
- file.project = @project
- file.branch = "new_branch_#{SecureRandom.hex(8)}"
- file.commit_message = 'Add new file'
- file.name = 'test.txt'
- file.content = "New file"
- end
- end.to raise_error(Resource::ApiFabricator::ResourceFabricationFailedError, /403 Forbidden/)
+ QA::Support::Retrier.retry_on_exception(max_attempts: 5, sleep_interval: 2) do
+ expect do
+ Resource::File.fabricate_via_api! do |file|
+ file.api_client = @user_api_client
+ file.project = @project
+ file.branch = "new_branch_#{SecureRandom.hex(8)}"
+ file.commit_message = 'Add new file'
+ file.name = 'test.txt'
+ file.content = "New file"
+ end
+ end.to raise_error(Resource::ApiFabricator::ResourceFabricationFailedError, /403 Forbidden/)
+ end
end
it 'is not allowed to commit via the API', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347865' do
- expect do
- Resource::Repository::Commit.fabricate_via_api! do |commit|
- commit.api_client = @user_api_client
- commit.project = @project
- commit.branch = "new_branch_#{SecureRandom.hex(8)}"
- commit.start_branch = @project.default_branch
- commit.commit_message = 'Add new file'
- commit.add_files([
- { file_path: 'test.txt', content: 'new file' }
- ])
- end
- end.to raise_error(Resource::ApiFabricator::ResourceFabricationFailedError, /403 Forbidden - You are not allowed to push into this branch/)
+ QA::Support::Retrier.retry_on_exception(max_attempts: 5, sleep_interval: 2) do
+ expect do
+ Resource::Repository::Commit.fabricate_via_api! do |commit|
+ commit.api_client = @user_api_client
+ commit.project = @project
+ commit.branch = "new_branch_#{SecureRandom.hex(8)}"
+ commit.start_branch = @project.default_branch
+ commit.commit_message = 'Add new file'
+ commit.add_files([
+ { file_path: 'test.txt', content: 'new file' }
+ ])
+ end
+ end.to raise_error(Resource::ApiFabricator::ResourceFabricationFailedError, /403 Forbidden - You are not allowed to push into this branch/)
+ end
end
end
diff --git a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb
index 895027a588d..bfb810b5c2b 100644
--- a/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb
+++ b/qa/qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb
@@ -1,11 +1,7 @@
# frozen_string_literal: true
module QA
- RSpec.describe 'Manage', :requires_admin, quarantine: {
- issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/350598',
- type: :needs_update,
- only: { subdomain: :staging }
- } do
+ RSpec.describe 'Manage', :requires_admin do
describe 'Add project member' do
before do
Runtime::Feature.enable(:invite_members_group_modal)
@@ -25,7 +21,7 @@ module QA
Page::Project::Menu.perform(&:click_members)
Page::Project::Members.perform do |members|
members.add_member(user.username)
-
+ members.search_member(user.username)
expect(members).to have_content("@#{user.username}")
end
end
diff --git a/qa/qa/support/page/logging.rb b/qa/qa/support/page/logging.rb
index f5299ed840d..b402639b843 100644
--- a/qa/qa/support/page/logging.rb
+++ b/qa/qa/support/page/logging.rb
@@ -4,6 +4,8 @@ module QA
module Support
module Page
module Logging
+ using Rainbow
+
def assert_no_element(name)
log("asserting no element :#{name}")
@@ -17,7 +19,7 @@ module QA
end
def scroll_to(selector, text: nil)
- msg = "scrolling to :#{selector}"
+ msg = "scrolling to :#{Rainbow(selector).underline.bright}"
msg += " with text: #{text}" if text
log(msg)
@@ -37,7 +39,7 @@ module QA
element = super
- log("found :#{name}") if element
+ log("found :#{Rainbow(name).underline.bright}")
element
end
@@ -47,7 +49,7 @@ module QA
elements = super
- log("found #{elements.size} :#{name}") if elements
+ log("found #{elements.size} :#{Rainbow(name).underline.bright}") if elements
elements
end
@@ -71,7 +73,7 @@ module QA
end
def click_element(name, page = nil, **kwargs)
- msg = ["clicking :#{name}"]
+ msg = ["clicking :#{Rainbow(name).underline.bright}"]
msg << ", expecting to be at #{page.class}" if page
msg << "with args #{kwargs}"
@@ -170,7 +172,7 @@ module QA
end
def log_has_element_or_not(method, name, found, **kwargs)
- msg = ["#{method} :#{name}"]
+ msg = ["#{method} :#{Rainbow(name).underline.bright}"]
msg << %Q(with text "#{kwargs[:text]}") if kwargs[:text]
msg << "class: #{kwargs[:class]}" if kwargs[:class]
msg << "(wait: #{kwargs[:wait] || Capybara.default_max_wait_time})"
diff --git a/qa/qa/support/repeater.rb b/qa/qa/support/repeater.rb
index a4e8035f964..1b9aa809051 100644
--- a/qa/qa/support/repeater.rb
+++ b/qa/qa/support/repeater.rb
@@ -1,10 +1,11 @@
# frozen_string_literal: true
-
require 'active_support/inflector'
+require 'rainbow/refinement'
module QA
module Support
module Repeater
+ using Rainbow
DEFAULT_MAX_WAIT_TIME = 60
RepeaterConditionExceededError = Class.new(RuntimeError)
@@ -39,7 +40,7 @@ module QA
QA::Runtime::Logger.debug(msg.join(' '))
end
- QA::Runtime::Logger.debug("Attempt number #{attempts + 1}") if log && max_attempts && attempts > 0
+ QA::Runtime::Logger.debug("Attempt number #{attempts + 1}".bg(:yellow).black) if log && max_attempts && attempts > 0
result = yield
if result
diff --git a/qa/qa/tools/initialize_gitlab_auth.rb b/qa/qa/tools/initialize_gitlab_auth.rb
index 86791f1f624..18e90f0d739 100644
--- a/qa/qa/tools/initialize_gitlab_auth.rb
+++ b/qa/qa/tools/initialize_gitlab_auth.rb
@@ -16,7 +16,7 @@ module QA
def run
Runtime::Scenario.define(:gitlab_address, address)
- puts "Signing in and creating the default password for the root user if it's not set already..."
+ QA::Runtime::Logger.info("Signing in and creating the default password for the root user if it's not set already...")
QA::Runtime::Browser.visit(:gitlab, QA::Page::Main::Login)
Flow::Login.sign_in
diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index 47791f76970..3be71d51a81 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -6,6 +6,7 @@ require 'securerandom'
require 'pathname'
require 'active_support/core_ext/hash'
require 'active_support/core_ext/object/blank'
+require 'rainbow/refinement'
require_relative 'qa_deprecation_toolkit_env'
QaDeprecationToolkitEnv.configure!
diff --git a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb
index 2cebf0d9473..087bfb197ec 100644
--- a/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb
+++ b/spec/lib/gitlab/pagination/keyset/in_operator_optimization/array_scope_columns_spec.rb
@@ -16,4 +16,13 @@ RSpec.describe Gitlab::Pagination::Keyset::InOperatorOptimization::ArrayScopeCol
it { expect { array_scope_columns }.to raise_error /No array columns were given/ }
end
+
+ context 'when Arel AS node is given as input' do
+ let(:scope) { Issue.select(Issue.arel_table[:id].as('id'), :title) }
+ let(:columns) { scope.select_values }
+
+ it 'works with Arel AS nodes' do
+ expect(array_scope_columns.array_aggregated_column_names).to eq(%w[array_cte_id_array array_cte_title_array])
+ end
+ end
end
diff --git a/spec/lib/gitlab/rack_attack/request_spec.rb b/spec/lib/gitlab/rack_attack/request_spec.rb
index 9b882f26480..4e304de3224 100644
--- a/spec/lib/gitlab/rack_attack/request_spec.rb
+++ b/spec/lib/gitlab/rack_attack/request_spec.rb
@@ -12,7 +12,7 @@ RSpec.describe Gitlab::RackAttack::Request do
::Rack::Attack::Request.new(
env.reverse_merge(
'REQUEST_METHOD' => 'GET',
- 'PATH_INFO' => path,
+ 'PATH_INFO' => Gitlab.config.gitlab.relative_url_root + path,
'rack.input' => StringIO.new,
'rack.session' => session
)
@@ -44,6 +44,14 @@ RSpec.describe Gitlab::RackAttack::Request do
with_them do
it { is_expected.to eq(expected) }
+
+ context 'when the application is mounted at a relative URL' do
+ before do
+ stub_config_setting(relative_url_root: '/gitlab/root')
+ end
+
+ it { is_expected.to eq(expected) }
+ end
end
end
@@ -65,6 +73,14 @@ RSpec.describe Gitlab::RackAttack::Request do
with_them do
it { is_expected.to eq(expected) }
+
+ context 'when the application is mounted at a relative URL' do
+ before do
+ stub_config_setting(relative_url_root: '/gitlab/root')
+ end
+
+ it { is_expected.to eq(expected) }
+ end
end
end
@@ -88,6 +104,14 @@ RSpec.describe Gitlab::RackAttack::Request do
with_them do
it { is_expected.to eq(expected) }
+
+ context 'when the application is mounted at a relative URL' do
+ before do
+ stub_config_setting(relative_url_root: '/gitlab/root')
+ end
+
+ it { is_expected.to eq(expected) }
+ end
end
end
@@ -107,6 +131,14 @@ RSpec.describe Gitlab::RackAttack::Request do
with_them do
it { is_expected.to eq(expected) }
+
+ context 'when the application is mounted at a relative URL' do
+ before do
+ stub_config_setting(relative_url_root: '/gitlab/root')
+ end
+
+ it { is_expected.to eq(expected) }
+ end
end
end
@@ -127,6 +159,14 @@ RSpec.describe Gitlab::RackAttack::Request do
with_them do
it { is_expected.to eq(expected) }
+
+ context 'when the application is mounted at a relative URL' do
+ before do
+ stub_config_setting(relative_url_root: '/gitlab/root')
+ end
+
+ it { is_expected.to eq(expected) }
+ end
end
end
@@ -162,6 +202,14 @@ RSpec.describe Gitlab::RackAttack::Request do
with_them do
it { is_expected.to eq(expected) }
+
+ context 'when the application is mounted at a relative URL' do
+ before do
+ stub_config_setting(relative_url_root: '/gitlab/root')
+ end
+
+ it { is_expected.to eq(expected) }
+ end
end
end
@@ -189,6 +237,14 @@ RSpec.describe Gitlab::RackAttack::Request do
with_them do
it { is_expected.to eq(expected) }
+
+ context 'when the application is mounted at a relative URL' do
+ before do
+ stub_config_setting(relative_url_root: '/gitlab/root')
+ end
+
+ it { is_expected.to eq(expected) }
+ end
end
end
@@ -255,6 +311,14 @@ RSpec.describe Gitlab::RackAttack::Request do
with_them do
it { is_expected.to eq(expected) }
+
+ context 'when the application is mounted at a relative URL' do
+ before do
+ stub_config_setting(relative_url_root: '/gitlab/root')
+ end
+
+ it { is_expected.to eq(expected) }
+ end
end
end
end
diff --git a/spec/models/loose_foreign_keys/deleted_record_spec.rb b/spec/models/loose_foreign_keys/deleted_record_spec.rb
index 07ffff746a5..23e0ed1f39d 100644
--- a/spec/models/loose_foreign_keys/deleted_record_spec.rb
+++ b/spec/models/loose_foreign_keys/deleted_record_spec.rb
@@ -6,15 +6,15 @@ RSpec.describe LooseForeignKeys::DeletedRecord, type: :model do
let_it_be(:table) { 'public.projects' }
describe 'class methods' do
- let_it_be(:deleted_record_1) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 5) }
- let_it_be(:deleted_record_2) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1) }
+ let_it_be(:deleted_record_1) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 5, cleanup_attempts: 2) }
+ let_it_be(:deleted_record_2) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1, cleanup_attempts: 0) }
let_it_be(:deleted_record_3) { described_class.create!(fully_qualified_table_name: 'public.other_table', primary_key_value: 3) }
- let_it_be(:deleted_record_4) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1) } # duplicate
+ let_it_be(:deleted_record_4) { described_class.create!(fully_qualified_table_name: table, primary_key_value: 1, cleanup_attempts: 1) } # duplicate
+
+ let(:records) { described_class.load_batch_for_table(table, 10) }
describe '.load_batch_for_table' do
it 'loads records and orders them by creation date' do
- records = described_class.load_batch_for_table(table, 10)
-
expect(records).to eq([deleted_record_1, deleted_record_2, deleted_record_4])
end
@@ -27,13 +27,38 @@ RSpec.describe LooseForeignKeys::DeletedRecord, type: :model do
describe '.mark_records_processed' do
it 'updates all records' do
- records = described_class.load_batch_for_table(table, 10)
described_class.mark_records_processed(records)
expect(described_class.status_pending.count).to eq(1)
expect(described_class.status_processed.count).to eq(3)
end
end
+
+ describe '.reschedule' do
+ it 'reschedules all records' do
+ time = Time.zone.parse('2022-01-01').utc
+ update_count = described_class.reschedule(records, time)
+
+ expect(update_count).to eq(records.size)
+
+ records.each(&:reload)
+
+ expect(records).to all(have_attributes(
+ cleanup_attempts: 0,
+ consume_after: time
+ ))
+ end
+ end
+
+ describe '.increment_attempts' do
+ it 'increaments the cleanup_attempts column' do
+ described_class.increment_attempts(records)
+
+ expect(deleted_record_1.reload.cleanup_attempts).to eq(3)
+ expect(deleted_record_2.reload.cleanup_attempts).to eq(1)
+ expect(deleted_record_4.reload.cleanup_attempts).to eq(2)
+ end
+ end
end
describe 'sliding_list partitioning' do
diff --git a/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb b/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb
index d3d57ea2444..538d9638879 100644
--- a/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb
+++ b/spec/services/loose_foreign_keys/batch_cleaner_service_spec.rb
@@ -115,4 +115,82 @@ RSpec.describe LooseForeignKeys::BatchCleanerService do
expect(loose_fk_child_table_2.where(parent_id_with_different_column: other_parent_record.id).count).to eq(2)
end
end
+
+ describe 'fair queueing' do
+ context 'when the execution is over the limit' do
+ let(:modification_tracker) { instance_double(LooseForeignKeys::ModificationTracker) }
+ let(:over_limit_return_values) { [true] }
+ let(:deleted_record) { LooseForeignKeys::DeletedRecord.load_batch_for_table('public._test_loose_fk_parent_table', 1).first }
+ let(:deleted_records_rescheduled_counter) { Gitlab::Metrics.registry.get(:loose_foreign_key_rescheduled_deleted_records) }
+ let(:deleted_records_incremented_counter) { Gitlab::Metrics.registry.get(:loose_foreign_key_incremented_deleted_records) }
+
+ let(:cleaner) do
+ described_class.new(parent_table: '_test_loose_fk_parent_table',
+ loose_foreign_key_definitions: loose_foreign_key_definitions,
+ deleted_parent_records: LooseForeignKeys::DeletedRecord.load_batch_for_table('public._test_loose_fk_parent_table', 100),
+ modification_tracker: modification_tracker
+ )
+ end
+
+ before do
+ parent_record_1.delete
+ allow(modification_tracker).to receive(:over_limit?).and_return(*over_limit_return_values)
+ allow(modification_tracker).to receive(:add_deletions)
+ end
+
+ context 'when the deleted record is under the maximum allowed cleanup attempts' do
+ it 'updates the cleanup_attempts column', :aggregate_failures do
+ deleted_record.update!(cleanup_attempts: 1)
+
+ cleaner.execute
+
+ expect(deleted_record.reload.cleanup_attempts).to eq(2)
+ expect(deleted_records_incremented_counter.get(table: loose_fk_parent_table.table_name, db_config_name: 'main')).to eq(1)
+ end
+
+ context 'when the deleted record is above the maximum allowed cleanup attempts' do
+ it 'reschedules the record', :aggregate_failures do
+ deleted_record.update!(cleanup_attempts: LooseForeignKeys::BatchCleanerService::CLEANUP_ATTEMPTS_BEFORE_RESCHEDULE + 1)
+
+ freeze_time do
+ cleaner.execute
+
+ expect(deleted_record.reload).to have_attributes(
+ cleanup_attempts: 0,
+ consume_after: 5.minutes.from_now
+ )
+ expect(deleted_records_rescheduled_counter.get(table: loose_fk_parent_table.table_name, db_config_name: 'main')).to eq(1)
+ end
+ end
+ end
+
+ describe 'when over limit happens on the second cleanup call without skip locked' do
+ # over_limit? is called twice, we test here the 2nd call
+ # - When invoking cleanup with SKIP LOCKED
+ # - When invoking cleanup (no SKIP LOCKED)
+ let(:over_limit_return_values) { [false, true] }
+
+ it 'updates the cleanup_attempts column' do
+ expect(cleaner).to receive(:run_cleaner_service).twice
+
+ deleted_record.update!(cleanup_attempts: 1)
+
+ cleaner.execute
+
+ expect(deleted_record.reload.cleanup_attempts).to eq(2)
+ end
+ end
+ end
+
+ context 'when the lfk_fair_queueing FF is off' do
+ before do
+ stub_feature_flags(lfk_fair_queueing: false)
+ end
+
+ it 'does nothing' do
+ expect { cleaner.execute }.not_to change { deleted_record.reload.cleanup_attempts }
+ end
+ end
+ end
+ end
end