diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 00:13:32 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 00:13:32 +0000 |
commit | d150848f7ed150d4f1887a4bff7cc4c423c55186 (patch) | |
tree | c43ccbe2e72615f7a60e647a4ce3ef2fa11dd35e | |
parent | ed28831dfe5fd267a9506cb6642e50fbcb581a7f (diff) | |
download | gitlab-ce-d150848f7ed150d4f1887a4bff7cc4c423c55186.tar.gz |
Add latest changes from gitlab-org/gitlab@master
25 files changed, 767 insertions, 168 deletions
diff --git a/.gitlab/ci/qa-report.gitlab-ci.yml b/.gitlab/ci/qa-report.gitlab-ci.yml new file mode 100644 index 00000000000..61cbcfd58da --- /dev/null +++ b/.gitlab/ci/qa-report.gitlab-ci.yml @@ -0,0 +1,15 @@ +test-reliability-report: + extends: + - .qa:rules:reliable-reports:schedule + image: + name: ${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_DEFAULT_BRANCH} + entrypoint: [""] + before_script: + - cd /home/gitlab/qa + script: + - echo "Generate report for 'staging-full' runs" + - bundle exec rake "reliable_spec_report[staging-full,30,true]" + - bundle exec rake "unreliable_spec_report[staging-full,30,true]" + - echo "Generate report for 'package-and-qa' runs" + - bundle exec rake "reliable_spec_report[package-and-qa,30,true]" + - bundle exec rake "unreliable_spec_report[package-and-qa,30,true]" diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 2c0b8122e3c..cf53e810d01 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -769,6 +769,11 @@ changes: *feature-flag-development-config-patterns allow_failure: true +.qa:rules:reliable-reports:schedule: + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $QA_RELIABLE_REPORT == "true"' + allow_failure: true + ############### # Rails rules # ############### diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 16ac9592801..397a7ea2e1d 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -57ac403e3ac18dbfa7e1e72bfb3eb8ab01cea626 +d69b465fdff3c04f8e3f395aed34aaa59f23fe76 diff --git a/app/services/ci/destroy_pipeline_service.rb b/app/services/ci/destroy_pipeline_service.rb index 476c7523d60..6fbde5d291c 100644 --- a/app/services/ci/destroy_pipeline_service.rb +++ b/app/services/ci/destroy_pipeline_service.rb @@ -12,7 +12,9 @@ module Ci # Ci::Pipeline#destroy triggers `use_fast_destroy :job_artifacts` and # ci_builds has ON DELETE CASCADE to ci_pipelines. The pipeline, the builds, # job and pipeline artifacts all get destroyed here. - pipeline.reset.destroy! + ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/345664') do + pipeline.reset.destroy! + end ServiceResponse.success(message: 'Pipeline not found') rescue ActiveRecord::RecordNotFound diff --git a/app/services/users/destroy_service.rb b/app/services/users/destroy_service.rb index 4ec875098fa..1634cc017ae 100644 --- a/app/services/users/destroy_service.rb +++ b/app/services/users/destroy_service.rb @@ -65,7 +65,10 @@ module Users user.destroy_dependent_associations_in_batches(exclude: [:snippets]) # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing - user_data = user.destroy + user_data = nil + ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/340260') do + user_data = user.destroy + end namespace.destroy user_data diff --git a/config/feature_flags/development/detect_cross_database_modification.yml b/config/feature_flags/development/detect_cross_database_modification.yml new file mode 100644 index 00000000000..7f74e136291 --- /dev/null +++ b/config/feature_flags/development/detect_cross_database_modification.yml @@ -0,0 +1,8 @@ +--- +name: detect_cross_database_modification +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73316 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344620 +milestone: '14.5' +type: development +group: group::sharding +default_enabled: false diff --git a/config/initializers/database_query_analyzers.rb b/config/initializers/database_query_analyzers.rb index a2db2d2a2b8..8a2fe1d8388 100644 --- a/config/initializers/database_query_analyzers.rb +++ b/config/initializers/database_query_analyzers.rb @@ -5,6 +5,10 @@ if Gitlab.dev_or_test_env? || Gitlab::Utils.to_boolean(ENV['GITLAB_ENABLE_QUERY_ Gitlab::Database::QueryAnalyzer.instance.hook! Gitlab::Database::QueryAnalyzer.instance.all_analyzers.append(::Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics) + if Rails.env.test? || Gitlab::Utils.to_boolean(ENV['ENABLE_CROSS_DATABASE_MODIFICATION_DETECTION'], default: false) + Gitlab::Database::QueryAnalyzer.instance.all_analyzers.append(::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification) + end + Gitlab::Application.configure do |config| config.middleware.use(Gitlab::Middleware::QueryAnalyzer) end diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index ad3b38c89af..5702c7a7dfd 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -2953,7 +2953,7 @@ GitLab can display the results of: - One or more reports in the merge request [code quality widget](../../user/project/merge_requests/code_quality.md#code-quality-widget). - Only one report in: - The merge request [diff annotations](../../user/project/merge_requests/code_quality.md#code-quality-in-diff-view). - Track progress on adding support for multiple reports in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/271077). + Track progress on adding support for multiple reports in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/328257). - The [full report](../metrics_reports.md). Track progress on adding support for multiple reports in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/9014). diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 987ff7c9fe8..86dc4c73062 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -535,7 +535,7 @@ to ensure the feature works properly. When using the testing environment, all feature flags are enabled by default. WARNING: -This does not apply to end-to-end (QA) tests, which [do not disable feature flags by default](#end-to-end-qa-tests). There is a different [process for using feature flags in end-to-end tests](../testing_guide/end_to_end/feature_flags.md). +This does not apply to end-to-end (QA) tests, which [do not enable feature flags by default](#end-to-end-qa-tests). There is a different [process for using feature flags in end-to-end tests](../testing_guide/end_to_end/feature_flags.md). To disable a feature flag in a test, use the `stub_feature_flags` helper. For example, to globally disable the `ci_live_trace` feature diff --git a/doc/operations/feature_flags.md b/doc/operations/feature_flags.md index ff07ade1646..2ef193b0f5d 100644 --- a/doc/operations/feature_flags.md +++ b/doc/operations/feature_flags.md @@ -418,20 +418,22 @@ connect to your project's feature flags, run the following command: ```shell docker run \ - -e UNLEASH_PROXY_SECRET=<secret> \ - -e UNLEASH_PROXY_URL=<project feature flags URL> \ - -e UNLEASH_PROXY_INSTANCE_ID=<project feature flags instance ID> \ - -e UNLEASH_PROXY_APP_NAME=<project environment> \ - -e UNLEASH_PROXY_API_TOKEN=<token> + -e UNLEASH_PROXY_SECRETS=<secret> \ + -e UNLEASH_URL=<project feature flags URL> \ + -e UNLEASH_INSTANCE_ID=<project feature flags instance ID> \ + -e UNLEASH_APP_NAME=<project environment> \ + -e UNLEASH_API_TOKEN=<tokenNotUsed> \ + -p 3000:3000 \ + unleashorg/unleash-proxy ``` | Variable | Value | | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| `UNLEASH_PROXY_SECRET` | Shared secret used to configure an [Unleash Proxy client](https://docs.getunleash.io/sdks/unleash-proxy#how-to-connect-to-the-proxy). | -| `UNLEASH_PROXY_URL` | Your project's API URL. For more details, read [Get access credentials](#get-access-credentials). | -| `UNLEASH_PROXY_INSTANCE_ID` | Your project's Instance ID. For more details, read [Get access credentials](#get-access-credentials). | -| `UNLEASH_PROXY_APP_NAME` | The name of the environment the application runs in. For more details, read [Get access credentials](#get-access-credentials). | -| `UNLEASH_PROXY_API_TOKEN` | Required to start the Unleash Proxy, but not used to connect to GitLab. Can be set to any value. | +| `UNLEASH_PROXY_SECRETS` | Shared secret used to configure an [Unleash Proxy client](https://docs.getunleash.io/sdks/unleash-proxy#how-to-connect-to-the-proxy). | +| `UNLEASH_URL` | Your project's API URL. For more details, read [Get access credentials](#get-access-credentials). | +| `UNLEASH_INSTANCE_ID` | Your project's Instance ID. For more details, read [Get access credentials](#get-access-credentials). | +| `UNLEASH_APP_NAME` | The name of the environment the application runs in. For more details, read [Get access credentials](#get-access-credentials). | +| `UNLEASH_API_TOKEN` | Required to start the Unleash Proxy, but not used to connect to GitLab. Can be set to any value. | ## Feature Flag Related Issues **(PREMIUM)** diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index c83436f24b4..9c74e5d2ca8 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -168,18 +168,6 @@ module Gitlab yield end - # This method will allow cross database modifications within the block - # Example: - # - # allow_cross_database_modification_within_transaction(url: 'url-to-an-issue') do - # create(:build) # inserts ci_build and project record in one transaction - # end - def self.allow_cross_database_modification_within_transaction(url:) - # this method will be overridden in: - # spec/support/database/cross_database_modification_check.rb - yield - end - def self.add_post_migrate_path_to_rails(force: false) return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 53de6697ca6..0f285688876 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -58,6 +58,8 @@ module Gitlab return unless parsed analyzers.each do |analyzer| + next if analyzer.suppressed? + analyzer.analyze(parsed) rescue StandardError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index d3a20523e74..e8066f7a706 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -4,16 +4,32 @@ module Gitlab module Database module QueryAnalyzers class Base + def self.suppressed? + Thread.current[self.suppress_key] + end + + def self.suppress=(value) + Thread.current[self.suppress_key] = value + end + + def self.with_suppressed(value = true, &blk) + previous = self.suppressed? + self.suppress = value + yield + ensure + self.suppress = previous + end + def self.begin! - Thread.current[self.class.name] = {} + Thread.current[self.context_key] = {} end def self.end! - Thread.current[self.class.name] = nil + Thread.current[self.context_key] = nil end def self.context - Thread.current[self.class.name] + Thread.current[self.context_key] end def self.enabled? @@ -23,6 +39,14 @@ module Gitlab def self.analyze(parsed) raise NotImplementedError end + + def self.context_key + "#{self.class.name}_context" + end + + def self.suppress_key + "#{self.class.name}_suppressed" + end end end end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb new file mode 100644 index 00000000000..2233f3c4646 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base + CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(StandardError) + + # This method will allow cross database modifications within the block + # Example: + # + # allow_cross_database_modification_within_transaction(url: 'url-to-an-issue') do + # create(:build) # inserts ci_build and project record in one transaction + # end + def self.allow_cross_database_modification_within_transaction(url:, &blk) + self.with_suppressed(true, &blk) + end + + # This method will prevent cross database modifications within the block + # if it was allowed previously + def self.with_cross_database_modification_prevented(&blk) + self.with_suppressed(false, &blk) + end + + def self.begin! + super + + context.merge!({ + transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 }, + modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new } + }) + end + + def self.enabled? + ::Feature::FlipperFeature.table_exists? && + Feature.enabled?(:detect_cross_database_modification, default_enabled: :yaml) + end + + # rubocop:disable Metrics/AbcSize + def self.analyze(parsed) + return if in_factory_bot_create? + + database = ::Gitlab::Database.db_config_name(parsed.connection) + sql = parsed.sql + + # We ignore BEGIN in tests as this is the outer transaction for + # DatabaseCleaner + if sql.start_with?('SAVEPOINT') || (!Rails.env.test? && sql.start_with?('BEGIN')) + context[:transaction_depth_by_db][database] += 1 + + return + elsif sql.start_with?('RELEASE SAVEPOINT', 'ROLLBACK TO SAVEPOINT') || (!Rails.env.test? && sql.start_with?('ROLLBACK', 'COMMIT')) + context[:transaction_depth_by_db][database] -= 1 + if context[:transaction_depth_by_db][database] <= 0 + context[:modified_tables_by_db][database].clear + end + + return + end + + return if context[:transaction_depth_by_db].values.all?(&:zero?) + + # PgQuery might fail in some cases due to limited nesting: + # https://github.com/pganalyze/pg_query/issues/209 + tables = sql.downcase.include?(' for update') ? parsed.pg.tables : parsed.pg.dml_tables + + # We have some code where plans and gitlab_subscriptions are lazily + # created and this causes lots of spec failures + # https://gitlab.com/gitlab-org/gitlab/-/issues/343394 + tables -= %w[plans gitlab_subscriptions] + + return if tables.empty? + + # All migrations will write to schema_migrations in the same transaction. + # It's safe to ignore this since schema_migrations exists in all + # databases + return if tables == ['schema_migrations'] + + context[:modified_tables_by_db][database].merge(tables) + all_tables = context[:modified_tables_by_db].values.map(&:to_a).flatten + schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables) + + if schemas.many? + message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ + "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables." \ + "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." + + if schemas.any? { |s| s.to_s.start_with?("undefined") } + message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ." + end + + raise CrossDatabaseModificationAcrossUnsupportedTablesError, message + end + rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e + ::Gitlab::ErrorTracking.track_exception(e, { gitlab_schemas: schemas, tables: all_tables, query: parsed.sql }) + raise if raise_exception? + end + # rubocop:enable Metrics/AbcSize + + # We only raise in tests for now otherwise some features will be broken + # in development. For now we've mostly only added allowlist based on + # spec names. Until we have allowed all the violations inline we don't + # want to raise in development. + def self.raise_exception? + Rails.env.test? + end + + # We ignore execution in the #create method from FactoryBot + # because it is not representative of real code we run in + # production. There are far too many false positives caused + # by instantiating objects in different `gitlab_schema` in a + # FactoryBot `create`. + def self.in_factory_bot_create? + Rails.env.test? && caller_locations.any? { |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' } + end + end + end + end +end diff --git a/qa/Gemfile b/qa/Gemfile index 692bee5cdaf..498d05b2254 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -25,6 +25,8 @@ gem 'octokit', '~> 4.21' gem 'webdrivers', '~> 5.0' gem 'zeitwerk', '~> 2.4' gem 'influxdb-client', '~> 1.17' +gem 'terminal-table', '~> 1.8', require: false +gem 'slack-notifier', '~> 2.4', require: false gem 'chemlab', '~> 0.9' gem 'chemlab-library-www-gitlab-com', '~> 0.1' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 733041524e9..2b5b5e368cf 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -27,7 +27,7 @@ GEM oj (>= 3.10, < 4) require_all (>= 2, < 4) uuid (>= 2.3, < 3) - ast (2.4.1) + ast (2.4.2) binding_ninja (0.2.3) byebug (9.1.0) capybara (3.35.3) @@ -141,7 +141,7 @@ GEM parallel (1.19.2) parallel_tests (2.29.0) parallel - parser (2.7.1.4) + parser (3.0.2.0) ast (~> 2.4.1) proc_to_ast (0.1.0) coderay @@ -203,6 +203,7 @@ GEM childprocess (>= 0.5, < 5.0) rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2) + slack-notifier (2.4.0) systemu (2.6.5) table_print (1.5.7) terminal-table (1.8.0) @@ -265,6 +266,8 @@ DEPENDENCIES rspec_junit_formatter (~> 0.4.1) ruby-debug-ide (~> 0.7.0) selenium-webdriver (~> 4.0) + slack-notifier (~> 2.4) + terminal-table (~> 1.8) timecop (~> 0.9.1) webdrivers (~> 5.0) zeitwerk (~> 2.4) diff --git a/qa/Rakefile b/qa/Rakefile index f24c81a9ec2..57360e98ca2 100644 --- a/qa/Rakefile +++ b/qa/Rakefile @@ -2,6 +2,7 @@ # rubocop:disable Rails/RakeEnvironment load 'tasks/webdrivers.rake' +load 'tasks/reliable_report.rake' require_relative 'qa/tools/revoke_all_personal_access_tokens' require_relative 'qa/tools/delete_subgroups' diff --git a/qa/qa/tools/reliable_report.rb b/qa/qa/tools/reliable_report.rb new file mode 100644 index 00000000000..9d2079171c1 --- /dev/null +++ b/qa/qa/tools/reliable_report.rb @@ -0,0 +1,234 @@ +# frozen_string_literal: true + +require "influxdb-client" +require "terminal-table" +require "slack-notifier" + +module QA + module Tools + class ReliableReport + def initialize(run_type, range = 30) + @results = 10 + @slack_channel = "#quality-reports" + @range = range + @run_type = run_type + @stable_title = "Top #{results} stable specs for past #{@range} days in '#{run_type}' runs" + @unstable_title = "Top #{results} unstable reliable specs for past #{@range} days in '#{run_type}' runs" + end + + # Print top stable specs + # + # @return [void] + def show_top_stable + puts terminal_table( + rows: top_stable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, + title: stable_title + ) + end + + # Post top stable spec report to slack + # Slice table in to multiple messages due to max char limitation + # + # @return [void] + def notify_top_stable + tables = top_stable.each_slice(5).map do |slice| + terminal_table( + rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] } + ) + end + + puts "\nSending top stable spec report to #{slack_channel} slack channel" + slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" } + notifier.post(text: "*#{stable_title}*", **slack_args) + tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) } + end + + # Print top unstable specs + # + # @return [void] + def show_top_unstable + return puts("No unstable tests present!") if top_unstable_reliable.empty? + + puts terminal_table( + rows: top_unstable_reliable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, + title: unstable_title + ) + end + + # Post top unstable reliable spec report to slack + # Slice table in to multiple messages due to max char limitation + # + # @return [void] + def notify_top_unstable + return puts("No unstable tests present!") if top_unstable_reliable.empty? + + tables = top_unstable_reliable.each_slice(5).map do |slice| + terminal_table( + rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] } + ) + end + + puts "\nSending top unstable reliable spec report to #{slack_channel} slack channel" + slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" } + notifier.post(text: "*#{unstable_title}*", **slack_args) + tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) } + end + + private + + attr_reader :results, + :slack_channel, + :range, + :run_type, + :stable_title, + :unstable_title + + # Top stable specs + # + # @return [Hash] + def top_stable + @top_stable ||= runs(reliable: false).sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h + end + + # Top unstable reliable specs + # + # @return [Hash] + def top_unstable_reliable + @top_unstable_reliable ||= runs(reliable: true) + .reject { |k, v| v[:failure_rate] == 0 } + .sort_by { |k, v| -v[:failure_rate] }[0..results - 1] + .to_h + end + + # Terminal table for result formatting + # + # @return [Terminal::Table] + def terminal_table(rows:, title: nil) + Terminal::Table.new( + headings: ["name", "runs", "failed", "failure rate"], + style: { all_separators: true }, + title: title, + rows: rows + ) + end + + # Spec parameters for table row + # + # @param [Array] parameters + # @return [Array] + def table_params(parameters) + [*parameters[1..2], "#{parameters.last}%"] + end + + # Name column value + # + # @param [String] name + # @param [String] file + # @return [String] + def name_column(name, file) + spec_name = name.length > 100 ? "#{name} ".scan(/.{1,100} /).map(&:strip).join("\n") : name + name_line = "name: '#{spec_name}'" + file_line = "file: '#{file}'" + + "#{name_line}\n#{file_line.ljust(110)}" + end + + # Test executions grouped by name + # + # @param [Boolean] reliable + # @return [Hash] + def runs(reliable:) + puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past 30 days in '#{run_type}' runs") + puts + + query_api.query(query: query(reliable)).values.each_with_object({}) do |table, result| + records = table.records + name = records.last.values["name"] + file = records.last.values["file_path"].split("/").last + runs = records.count + failed = records.count { |r| r.values["status"] == "failed" } + failure_rate = (failed.to_f / runs.to_f) * 100 + + result[name] = { + file: file, + runs: runs, + failed: failed, + failure_rate: failure_rate == 0 ? failure_rate.round(0) : failure_rate.round(2) + } + end + end + + # Flux query + # + # @param [Boolean] reliable + # @return [String] + def query(reliable) + <<~QUERY + from(bucket: "e2e-test-stats") + |> range(start: -#{range}d) + |> filter(fn: (r) => r._measurement == "test-stats" and + r.run_type == "#{run_type}" and + r.status != "pending" and + r.merge_request == "false" and + r.quarantined == "false" and + r.reliable == "#{reliable}" and + r._field == "id" + ) + |> group(columns: ["name"]) + QUERY + end + + # Query client + # + # @return [QueryApi] + def query_api + @query_api ||= influx_client.create_query_api + end + + # InfluxDb client + # + # @return [InfluxDB2::Client] + def influx_client + @influx_client ||= InfluxDB2::Client.new( + influxdb_url, + influxdb_token, + bucket: "e2e-test-stats", + org: "gitlab-qa", + precision: InfluxDB2::WritePrecision::NANOSECOND + ) + end + + # Slack notifier + # + # @return [Slack::Notifier] + def notifier + @notifier ||= Slack::Notifier.new( + slack_webhook_url, + channel: slack_channel, + username: "Reliable spec reporter" + ) + end + + # InfluxDb instance url + # + # @return [String] + def influxdb_url + @influxdb_url ||= ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL environment variable") + end + + # Influxdb token + # + # @return [String] + def influxdb_token + @influxdb_token ||= ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN environment variable") + end + + # Slack webhook url + # + # @return [String] + def slack_webhook_url + @slack_webhook_url ||= ENV["CI_SLACK_WEBHOOK_URL"] || raise("Missing CI_SLACK_WEBHOOK_URL environment variable") + end + end + end +end diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb new file mode 100644 index 00000000000..c7d4d28fb21 --- /dev/null +++ b/qa/spec/tools/reliable_report_spec.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +describe QA::Tools::ReliableReport do + include QA::Support::Helpers::StubEnv + + subject(:reporter) { described_class.new(run_type, range) } + + let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) } + let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) } + let(:query_api) { instance_double("InfluxDB2::QueryApi") } + + let(:slack_channel) { "#quality-reports" } + let(:run_type) { "package-and-qa" } + let(:range) { 30 } + let(:results) { 10 } + + let(:runs) { { 0 => stable_spec, 1 => unstable_spec } } + + let(:stable_spec) do + spec_values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb" } + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: spec_values), + instance_double("InfluxDB2::FluxRecord", values: spec_values), + instance_double("InfluxDB2::FluxRecord", values: spec_values) + ] + ) + end + + let(:unstable_spec) do + spec_values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb" } + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: { **spec_values, "status" => "passed" }), + instance_double("InfluxDB2::FluxRecord", values: spec_values), + instance_double("InfluxDB2::FluxRecord", values: spec_values) + ] + ) + end + + def flux_query(reliable) + <<~QUERY + from(bucket: "e2e-test-stats") + |> range(start: -#{range}d) + |> filter(fn: (r) => r._measurement == "test-stats" and + r.run_type == "#{run_type}" and + r.status != "pending" and + r.merge_request == "false" and + r.quarantined == "false" and + r.reliable == "#{reliable}" and + r._field == "id" + ) + |> group(columns: ["name"]) + QUERY + end + + def table(rows, title = nil) + Terminal::Table.new( + headings: ["name", "runs", "failed", "failure rate"], + style: { all_separators: true }, + title: title, + rows: rows + ) + end + + def name_column(spec_name) + name = "name: '#{spec_name}'" + file = "file: 'spec.rb'".ljust(110) + + "#{name}\n#{file}" + end + + before do + stub_env("QA_INFLUXDB_URL", "url") + stub_env("QA_INFLUXDB_TOKEN", "token") + stub_env("CI_SLACK_WEBHOOK_URL", "slack_url") + + allow(Slack::Notifier).to receive(:new).and_return(slack_notifier) + allow(InfluxDB2::Client).to receive(:new).and_return(influx_client) + allow(query_api).to receive(:query).with(query: query).and_return(runs) + end + + context "with stable spec report" do + let(:query) { flux_query(false) } + let(:fetch_message) { "Fetching data on test execution for past #{range} days in '#{run_type}' runs" } + let(:slack_send_message) { "Sending top stable spec report to #{slack_channel} slack channel" } + let(:title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" } + let(:rows) do + [ + [name_column("stable spec"), 3, 0, "0%"], + [name_column("unstable spec"), 3, 2, "66.67%"] + ] + end + + it "prints top stable spec report to console" do + expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout + end + + it "sends top stable spec report to slack" do + slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" } + + expect { reporter.notify_top_stable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout + expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args) + expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args) + end + end + + context "with unstable spec report" do + let(:query) { flux_query(true) } + let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" } + let(:slack_send_message) { "Sending top unstable reliable spec report to #{slack_channel} slack channel" } + let(:title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" } + let(:rows) { [[name_column("unstable spec"), 3, 2, "66.67%"]] } + + it "prints top unstable spec report to console" do + expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout + end + + it "sends top unstable reliable spec report to slack" do + slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" } + + expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout + expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args) + expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args) + end + end + + context "without unstable reliable specs" do + let(:query) { flux_query(true) } + let(:runs) { { 0 => stable_spec } } + let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" } + let(:no_result_message) { "No unstable tests present!" } + + it "prints no result message to console" do + expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout + end + + it "skips slack notification" do + expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout + expect(slack_notifier).not_to have_received(:post) + end + end +end diff --git a/qa/tasks/reliable_report.rake b/qa/tasks/reliable_report.rake new file mode 100644 index 00000000000..204c959093a --- /dev/null +++ b/qa/tasks/reliable_report.rake @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Rails/RakeEnvironment + +require_relative "../qa/tools/reliable_report" + +desc "Fetch top most reliable specs" +task :reliable_spec_report, [:run_type, :range, :create_slack_report] do |_task, args| + report = QA::Tools::ReliableReport.new(args[:run_type] || "package-and-qa", args[:range]) + + report.show_top_stable + report.notify_top_stable if args[:create_slack_report] == 'true' +end + +desc "Fetch top most unstable reliable specs" +task :unreliable_spec_report, [:run_type, :range, :create_slack_report] do |_task, args| + report = QA::Tools::ReliableReport.new(args[:run_type] || "package-and-qa", args[:range]) + + report.show_top_unstable + report.notify_top_unstable if args[:create_slack_report] == 'true' +end +# rubocop:enable Rails/RakeEnvironment diff --git a/spec/lib/gitlab/database/query_analyzer_spec.rb b/spec/lib/gitlab/database/query_analyzer_spec.rb index 16b73b8123f..82a1c7143d5 100644 --- a/spec/lib/gitlab/database/query_analyzer_spec.rb +++ b/spec/lib/gitlab/database/query_analyzer_spec.rb @@ -9,6 +9,7 @@ RSpec.describe Gitlab::Database::QueryAnalyzer, query_analyzers: false do before do allow(described_class.instance).to receive(:all_analyzers).and_return([analyzer, disabled_analyzer]) allow(analyzer).to receive(:enabled?).and_return(true) + allow(analyzer).to receive(:suppressed?).and_return(false) allow(analyzer).to receive(:begin!) allow(analyzer).to receive(:end!) allow(disabled_analyzer).to receive(:enabled?).and_return(false) @@ -125,6 +126,13 @@ RSpec.describe Gitlab::Database::QueryAnalyzer, query_analyzers: false do expect { process_sql("SELECT 1 FROM projects") }.not_to raise_error end + it 'does not call analyze on suppressed analyzers' do + expect(analyzer).to receive(:suppressed?).and_return(true) + expect(analyzer).not_to receive(:analyze) + + expect { process_sql("SELECT 1 FROM projects") }.not_to raise_error + end + def process_sql(sql) described_class.instance.within do ApplicationRecord.load_balancer.read_write do |connection| diff --git a/spec/support_specs/database/prevent_cross_database_modification_spec.rb b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb index 365f1f32858..eb8ccb0bd89 100644 --- a/spec/support_specs/database/prevent_cross_database_modification_spec.rb +++ b/spec/lib/gitlab/database/query_analyzers/prevent_cross_database_modification_spec.rb @@ -2,10 +2,18 @@ require 'spec_helper' -RSpec.describe 'Database::PreventCrossDatabaseModification' do +RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification, query_analyzers: false do let_it_be(:pipeline, refind: true) { create(:ci_pipeline) } let_it_be(:project, refind: true) { create(:project) } + before do + allow(Gitlab::Database::QueryAnalyzer.instance).to receive(:all_analyzers).and_return([described_class]) + end + + around do |example| + Gitlab::Database::QueryAnalyzer.instance.within { example.run } + end + shared_examples 'successful examples' do context 'outside transaction' do it { expect { run_queries }.not_to raise_error } @@ -122,10 +130,10 @@ RSpec.describe 'Database::PreventCrossDatabaseModification' do include_examples 'successful examples' end - describe '#allow_cross_database_modification_within_transaction' do + describe '.allow_cross_database_modification_within_transaction' do it 'skips raising error' do expect do - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'gitlab-issue') do + described_class.allow_cross_database_modification_within_transaction(url: 'gitlab-issue') do Project.transaction do pipeline.touch project.touch @@ -136,7 +144,7 @@ RSpec.describe 'Database::PreventCrossDatabaseModification' do it 'skips raising error on factory creation' do expect do - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'gitlab-issue') do + described_class.allow_cross_database_modification_within_transaction(url: 'gitlab-issue') do ApplicationRecord.transaction do create(:ci_pipeline) end diff --git a/spec/lib/gitlab/middleware/query_analyzer_spec.rb b/spec/lib/gitlab/middleware/query_analyzer_spec.rb new file mode 100644 index 00000000000..5ebe6a92da6 --- /dev/null +++ b/spec/lib/gitlab/middleware/query_analyzer_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Middleware::QueryAnalyzer, query_analyzers: false do + describe 'the PreventCrossDatabaseModification' do + describe '#call' do + let(:app) { double(:app) } + let(:middleware) { described_class.new(app) } + let(:env) { {} } + + subject { middleware.call(env) } + + context 'when there is a cross modification' do + before do + allow(app).to receive(:call) do + Project.transaction do + Project.where(id: -1).update_all(id: -1) + ::Ci::Pipeline.where(id: -1).update_all(id: -1) + end + end + end + + it 'detects cross modifications and tracks exception' do + expect(::Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + + expect { subject }.not_to raise_error + end + + context 'when the detect_cross_database_modification is disabled' do + before do + stub_feature_flags(detect_cross_database_modification: false) + end + + it 'does not detect cross modifications' do + expect(::Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + subject + end + end + end + + context 'when there is no cross modification' do + before do + allow(app).to receive(:call) do + Project.transaction do + Project.where(id: -1).update_all(id: -1) + Namespace.where(id: -1).update_all(id: -1) + end + end + end + + it 'does not log anything' do + expect(::Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + subject + end + end + end + end +end diff --git a/spec/lib/gitlab/sidekiq_middleware/query_analyzer_spec.rb b/spec/lib/gitlab/sidekiq_middleware/query_analyzer_spec.rb new file mode 100644 index 00000000000..e58af1d60fe --- /dev/null +++ b/spec/lib/gitlab/sidekiq_middleware/query_analyzer_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::SidekiqMiddleware::QueryAnalyzer, query_analyzers: false do + describe 'the PreventCrossDatabaseModification' do + describe '#call' do + let(:worker) { double(:worker) } + let(:job) { { 'jid' => 'job123' } } + let(:queue) { 'some-queue' } + let(:middleware) { described_class.new } + + def do_queries + end + + subject { middleware.call(worker, job, queue) { do_queries } } + + context 'when there is a cross modification' do + def do_queries + Project.transaction do + Project.where(id: -1).update_all(id: -1) + ::Ci::Pipeline.where(id: -1).update_all(id: -1) + end + end + + it 'detects cross modifications and tracks exception' do + expect(::Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + + subject + end + + context 'when the detect_cross_database_modification is disabled' do + before do + stub_feature_flags(detect_cross_database_modification: false) + end + + it 'does not detect cross modifications' do + expect(::Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + subject + end + end + end + + context 'when there is no cross modification' do + def do_queries + Project.transaction do + Project.where(id: -1).update_all(id: -1) + Namespace.where(id: -1).update_all(id: -1) + end + end + + it 'does not log anything' do + expect(::Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception) + + subject + end + end + end + end +end diff --git a/spec/support/database/prevent_cross_database_modification.rb b/spec/support/database/prevent_cross_database_modification.rb index 09a87f92072..c509aecf9b8 100644 --- a/spec/support/database/prevent_cross_database_modification.rb +++ b/spec/support/database/prevent_cross_database_modification.rb @@ -1,148 +1,31 @@ # frozen_string_literal: true -module Database - module PreventCrossDatabaseModification - CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(StandardError) - - module GitlabDatabaseMixin - def allow_cross_database_modification_within_transaction(url:) - cross_database_context = Database::PreventCrossDatabaseModification.cross_database_context - return yield unless cross_database_context && cross_database_context[:enabled] - - transaction_tracker_enabled_was = cross_database_context[:enabled] - cross_database_context[:enabled] = false - - yield - ensure - cross_database_context[:enabled] = transaction_tracker_enabled_was if cross_database_context - end - end - - module SpecHelpers - def with_cross_database_modification_prevented - subscriber = ActiveSupport::Notifications.subscribe('sql.active_record') do |name, start, finish, id, payload| - PreventCrossDatabaseModification.prevent_cross_database_modification!(payload[:connection], payload[:sql]) - end - - PreventCrossDatabaseModification.reset_cross_database_context! - PreventCrossDatabaseModification.cross_database_context.merge!(enabled: true, subscriber: subscriber) - - yield if block_given? - ensure - cleanup_with_cross_database_modification_prevented if block_given? - end - - def cleanup_with_cross_database_modification_prevented - if PreventCrossDatabaseModification.cross_database_context - ActiveSupport::Notifications.unsubscribe(PreventCrossDatabaseModification.cross_database_context[:subscriber]) - PreventCrossDatabaseModification.cross_database_context[:enabled] = false - end - end - end - - def self.cross_database_context - Thread.current[:transaction_tracker] - end - - def self.reset_cross_database_context! - Thread.current[:transaction_tracker] = initial_data - end - - def self.initial_data - { - enabled: false, - transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 }, - modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new } - } - end - - def self.prevent_cross_database_modification!(connection, sql) - return unless cross_database_context - return unless cross_database_context[:enabled] - - return if connection.pool.instance_of?(ActiveRecord::ConnectionAdapters::NullPool) - return if in_factory_bot_create? - - database = connection.pool.db_config.name - - if sql.start_with?('SAVEPOINT') - cross_database_context[:transaction_depth_by_db][database] += 1 - - return - elsif sql.start_with?('RELEASE SAVEPOINT', 'ROLLBACK TO SAVEPOINT') - cross_database_context[:transaction_depth_by_db][database] -= 1 - if cross_database_context[:transaction_depth_by_db][database] <= 0 - cross_database_context[:modified_tables_by_db][database].clear - end - - return - end - - return if cross_database_context[:transaction_depth_by_db].values.all?(&:zero?) - - # PgQuery might fail in some cases due to limited nesting: - # https://github.com/pganalyze/pg_query/issues/209 - parsed_query = PgQuery.parse(sql) - tables = sql.downcase.include?(' for update') ? parsed_query.tables : parsed_query.dml_tables - - # We have some code where plans and gitlab_subscriptions are lazily - # created and this causes lots of spec failures - # https://gitlab.com/gitlab-org/gitlab/-/issues/343394 - tables -= %w[plans gitlab_subscriptions] - - return if tables.empty? - - # All migrations will write to schema_migrations in the same transaction. - # It's safe to ignore this since schema_migrations exists in all - # databases - return if tables == ['schema_migrations'] - - cross_database_context[:modified_tables_by_db][database].merge(tables) - - all_tables = cross_database_context[:modified_tables_by_db].values.map(&:to_a).flatten - schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables) - - if schemas.many? - message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ - "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables." \ - "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." - - if schemas.any? { |s| s.to_s.start_with?("undefined") } - message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to spec/support/database/gitlab_schemas.yml ." - end - - raise Database::PreventCrossDatabaseModification::CrossDatabaseModificationAcrossUnsupportedTablesError, message - end - end - - # We ignore execution in the #create method from FactoryBot - # because it is not representative of real code we run in - # production. There are far too many false positives caused - # by instantiating objects in different `gitlab_schema` in a - # FactoryBot `create`. - def self.in_factory_bot_create? - caller_locations.any? { |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' } - end - end +module PreventCrossDatabaseModificationSpecHelpers + delegate :with_cross_database_modification_prevented, + :allow_cross_database_modification_within_transaction, + to: :'::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification' end -Gitlab::Database.singleton_class.prepend( - Database::PreventCrossDatabaseModification::GitlabDatabaseMixin) - CROSS_DB_MODIFICATION_ALLOW_LIST = Set.new(YAML.load_file(File.join(__dir__, 'cross-database-modification-allowlist.yml'))).freeze RSpec.configure do |config| - config.include(::Database::PreventCrossDatabaseModification::SpecHelpers) + config.include(PreventCrossDatabaseModificationSpecHelpers) + + # By default allow cross-modifications as we want to observe only transactions + # within a specific block of execution which is defined be `before(:each)` and `after(:each)` + config.before(:all) do + ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.suppress = true + end # Using before and after blocks because the around block causes problems with the let_it_be # record creations. It makes an extra savepoint which breaks the transaction count logic. config.before do |example_file| - if CROSS_DB_MODIFICATION_ALLOW_LIST.exclude?(example_file.file_path_rerun_argument) - with_cross_database_modification_prevented - end + ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.suppress = + CROSS_DB_MODIFICATION_ALLOW_LIST.include?(example_file.file_path_rerun_argument) end + # Reset after execution to preferred state config.after do |example_file| - cleanup_with_cross_database_modification_prevented + ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.suppress = true end end |