diff options
27 files changed, 452 insertions, 117 deletions
diff --git a/.rubocop_todo/lint/unused_block_argument.yml b/.rubocop_todo/lint/unused_block_argument.yml index bf3bde843af..a70c3823c1d 100644 --- a/.rubocop_todo/lint/unused_block_argument.yml +++ b/.rubocop_todo/lint/unused_block_argument.yml @@ -122,7 +122,6 @@ Lint/UnusedBlockArgument: - 'ee/lib/tasks/gitlab/elastic.rake' - 'ee/lib/tasks/gitlab/indexer.rake' - 'ee/lib/tasks/gitlab/seed/insights.rake' - - 'ee/spec/config/metrics/every_metric_definition_spec.rb' - 'ee/spec/elastic/migrate/20220118150500_delete_orphaned_commits_spec.rb' - 'ee/spec/elastic/migrate/20220119120500_populate_commit_permissions_in_main_index_spec.rb' - 'ee/spec/elastic/migrate/20220613120500_migrate_commits_to_separate_index_spec.rb' diff --git a/app/services/metrics/global_metrics_update_service.rb b/app/services/metrics/global_metrics_update_service.rb new file mode 100644 index 00000000000..356de58ba2e --- /dev/null +++ b/app/services/metrics/global_metrics_update_service.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Metrics + # Update metrics regarding GitLab instance wide + # + # Anything that is not specific to a machine, process, request or any other context + # can be updated from this services. + # + # Examples of metrics that qualify: + # * Global counters (instance users, instance projects...) + # * State of settings stored in the database (whether a feature is active or not, tuning values...) + # + class GlobalMetricsUpdateService + def execute + return unless ::Gitlab::Metrics.prometheus_metrics_enabled? + + maintenance_mode_metric.set({}, (::Gitlab.maintenance_mode? ? 1 : 0)) + end + + def maintenance_mode_metric + ::Gitlab::Metrics.gauge(:gitlab_maintenance_mode, 'Is GitLab Maintenance Mode enabled?') + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 89f8867fe62..6c3b4990023 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -543,6 +543,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:metrics_global_metrics_update + :worker_name: Metrics::GlobalMetricsUpdateWorker + :feature_category: :metrics + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:namespaces_in_product_marketing_emails :worker_name: Namespaces::InProductMarketingEmailsWorker :feature_category: :experimentation_activation diff --git a/app/workers/metrics/global_metrics_update_worker.rb b/app/workers/metrics/global_metrics_update_worker.rb new file mode 100644 index 00000000000..326403a2f8f --- /dev/null +++ b/app/workers/metrics/global_metrics_update_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Metrics + class GlobalMetricsUpdateWorker + include ApplicationWorker + + idempotent! + data_consistency :sticky + feature_category :metrics + + include ExclusiveLeaseGuard + # rubocop:disable Scalability/CronWorkerContext + # This worker does not perform work scoped to a context + include CronjobQueue + # rubocop:enable Scalability/CronWorkerContext + + LEASE_TIMEOUT = 2.minutes + + def perform + try_obtain_lease { ::Metrics::GlobalMetricsUpdateService.new.execute } + end + + def lease_timeout + LEASE_TIMEOUT + end + end +end diff --git a/config/feature_flags/development/limited_commit_parser.yml b/config/feature_flags/development/limited_commit_parser.yml new file mode 100644 index 00000000000..026d7824aaf --- /dev/null +++ b/config/feature_flags/development/limited_commit_parser.yml @@ -0,0 +1,8 @@ +--- +name: limited_commit_parser +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117188 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/406461 +milestone: '15.11' +type: development +group: group::source code +default_enabled: false diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 24e5a93581a..2b31c6c55d7 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -685,6 +685,9 @@ Settings.cron_jobs['ci_runners_stale_machines_cleanup_worker']['job_class'] = 'C Settings.cron_jobs['cleanup_dangling_debian_package_files_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['cleanup_dangling_debian_package_files_worker']['cron'] ||= '20 21 * * *' Settings.cron_jobs['cleanup_dangling_debian_package_files_worker']['job_class'] = 'Packages::Debian::CleanupDanglingPackageFilesWorker' +Settings.cron_jobs['global_metrics_update_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['global_metrics_update_worker']['cron'] ||= '*/1 * * * *' +Settings.cron_jobs['global_metrics_update_worker']['job_class'] ||= 'Metrics::GlobalMetricsUpdateWorker' Gitlab.ee do Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({}) diff --git a/config/metrics/settings/20230328165533_mode.yml b/config/metrics/settings/20230328165533_mode.yml new file mode 100644 index 00000000000..bc8019d86aa --- /dev/null +++ b/config/metrics/settings/20230328165533_mode.yml @@ -0,0 +1,23 @@ +--- +key_path: database.mode +description: What PostgreSQL database mode is used. Possible values are + "single-database", "single-database-ci-connection", or "multiple-databases" +product_section: enablement +product_stage: enablement +product_group: database +product_category: database +value_type: string +status: active +milestone: "15.11" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115900 +time_frame: none +instrumentation_class: DatabaseMode +data_source: system +data_category: optional +distribution: +- ce +- ee +tier: +- free +- premium +- ultimate diff --git a/db/post_migrate/20230403041917_swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com.rb b/db/post_migrate/20230403041917_swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com.rb index e2a08276343..fb941e1cb51 100644 --- a/db/post_migrate/20230403041917_swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com.rb +++ b/db/post_migrate/20230403041917_swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com.rb @@ -1,83 +1,8 @@ # frozen_string_literal: true class SwapIssueUserMentionsNoteIdToBigintForGitlabDotCom < Gitlab::Database::Migration[2.1] - include Gitlab::Database::MigrationHelpers::ConvertToBigint + # No-op, moved to db/post_migrate/20230413041917_swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com_2.rb + def up; end - disable_ddl_transaction! - - TABLE_NAME = 'issue_user_mentions' - - def up - return unless should_run? - - swap - end - - def down - return unless should_run? - - swap - - add_concurrent_index TABLE_NAME, :note_id_convert_to_bigint, unique: true, - name: 'index_issue_user_mentions_on_note_id_convert_to_bigint', - where: 'note_id_convert_to_bigint IS NOT NULL' - - add_concurrent_foreign_key TABLE_NAME, :notes, column: :note_id_convert_to_bigint, - name: 'fk_issue_user_mentions_note_id_convert_to_bigint', - on_delete: :cascade, - validate: false - end - - def swap - # This will replace the existing index_issue_user_mentions_on_note_id - add_concurrent_index TABLE_NAME, :note_id_convert_to_bigint, unique: true, - name: 'index_issue_user_mentions_on_note_id_convert_to_bigint', - where: 'note_id_convert_to_bigint IS NOT NULL' - - # This will replace the existing issue_user_mentions_on_issue_id_and_note_id_index - add_concurrent_index TABLE_NAME, [:issue_id, :note_id_convert_to_bigint], unique: true, - name: 'tmp_issue_user_mentions_on_issue_id_and_note_id_index' - - # This will replace the existing issue_user_mentions_on_issue_id_index - add_concurrent_index TABLE_NAME, :issue_id, unique: true, - name: 'tmp_issue_user_mentions_on_issue_id_index', - where: 'note_id_convert_to_bigint IS NULL' - - # This will replace the existing fk_rails_3861d9fefa - add_concurrent_foreign_key TABLE_NAME, :notes, column: :note_id_convert_to_bigint, - name: 'fk_issue_user_mentions_note_id_convert_to_bigint', - on_delete: :cascade - - with_lock_retries(raise_on_exhaustion: true) do - execute "LOCK TABLE notes, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" - - execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id TO note_id_tmp" - execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id_convert_to_bigint TO note_id" - execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id_tmp TO note_id_convert_to_bigint" - - function_name = Gitlab::Database::UnidirectionalCopyTrigger - .on_table(TABLE_NAME, connection: connection) - .name(:note_id, :note_id_convert_to_bigint) - execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" - - execute 'DROP INDEX IF EXISTS index_issue_user_mentions_on_note_id' - rename_index TABLE_NAME, 'index_issue_user_mentions_on_note_id_convert_to_bigint', - 'index_issue_user_mentions_on_note_id' - - execute 'DROP INDEX IF EXISTS issue_user_mentions_on_issue_id_and_note_id_index' - rename_index TABLE_NAME, 'tmp_issue_user_mentions_on_issue_id_and_note_id_index', - 'issue_user_mentions_on_issue_id_and_note_id_index' - - execute 'DROP INDEX IF EXISTS issue_user_mentions_on_issue_id_index' - rename_index TABLE_NAME, 'tmp_issue_user_mentions_on_issue_id_index', - 'issue_user_mentions_on_issue_id_index' - - execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT IF EXISTS fk_rails_3861d9fefa" - rename_constraint(TABLE_NAME, 'fk_issue_user_mentions_note_id_convert_to_bigint', 'fk_rails_3861d9fefa') - end - end - - def should_run? - com_or_dev_or_test_but_not_jh? - end + def down; end end diff --git a/db/post_migrate/20230413041917_swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com_2.rb b/db/post_migrate/20230413041917_swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com_2.rb new file mode 100644 index 00000000000..cb29b47a64d --- /dev/null +++ b/db/post_migrate/20230413041917_swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com_2.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +class SwapIssueUserMentionsNoteIdToBigintForGitlabDotCom2 < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::ConvertToBigint + + disable_ddl_transaction! + + TABLE_NAME = 'issue_user_mentions' + + def up + return unless should_run? + return if columns_alredy_swapped? + + swap + end + + def down + return unless should_run? + return unless columns_alredy_swapped? + + swap + + add_concurrent_index TABLE_NAME, :note_id_convert_to_bigint, unique: true, + name: 'index_issue_user_mentions_on_note_id_convert_to_bigint', + where: 'note_id_convert_to_bigint IS NOT NULL' + + add_concurrent_foreign_key TABLE_NAME, :notes, column: :note_id_convert_to_bigint, + name: 'fk_issue_user_mentions_note_id_convert_to_bigint', + on_delete: :cascade, + validate: false + end + + def swap + # This will replace the existing index_issue_user_mentions_on_note_id + add_concurrent_index TABLE_NAME, :note_id_convert_to_bigint, unique: true, + name: 'index_issue_user_mentions_on_note_id_convert_to_bigint', + where: 'note_id_convert_to_bigint IS NOT NULL' + + # This will replace the existing issue_user_mentions_on_issue_id_and_note_id_index + add_concurrent_index TABLE_NAME, [:issue_id, :note_id_convert_to_bigint], unique: true, + name: 'tmp_issue_user_mentions_on_issue_id_and_note_id_index' + + # This will replace the existing issue_user_mentions_on_issue_id_index + add_concurrent_index TABLE_NAME, :issue_id, unique: true, + name: 'tmp_issue_user_mentions_on_issue_id_index', + where: 'note_id_convert_to_bigint IS NULL' + + # This will replace the existing fk_rails_3861d9fefa + add_concurrent_foreign_key TABLE_NAME, :notes, column: :note_id_convert_to_bigint, + name: 'fk_issue_user_mentions_note_id_convert_to_bigint', + on_delete: :cascade + + with_lock_retries(raise_on_exhaustion: true) do + execute "LOCK TABLE notes, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id TO note_id_tmp" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id_convert_to_bigint TO note_id" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id_tmp TO note_id_convert_to_bigint" + + function_name = Gitlab::Database::UnidirectionalCopyTrigger + .on_table(TABLE_NAME, connection: connection) + .name(:note_id, :note_id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + execute 'DROP INDEX IF EXISTS index_issue_user_mentions_on_note_id' + rename_index TABLE_NAME, 'index_issue_user_mentions_on_note_id_convert_to_bigint', + 'index_issue_user_mentions_on_note_id' + + execute 'DROP INDEX IF EXISTS issue_user_mentions_on_issue_id_and_note_id_index' + rename_index TABLE_NAME, 'tmp_issue_user_mentions_on_issue_id_and_note_id_index', + 'issue_user_mentions_on_issue_id_and_note_id_index' + + execute 'DROP INDEX IF EXISTS issue_user_mentions_on_issue_id_index' + rename_index TABLE_NAME, 'tmp_issue_user_mentions_on_issue_id_index', + 'issue_user_mentions_on_issue_id_index' + + execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT IF EXISTS fk_rails_3861d9fefa" + rename_constraint(TABLE_NAME, 'fk_issue_user_mentions_note_id_convert_to_bigint', 'fk_rails_3861d9fefa') + end + end + + def should_run? + com_or_dev_or_test_but_not_jh? + end + + def columns_alredy_swapped? + table_columns = columns(TABLE_NAME) + note_id = table_columns.find { |c| c.name == 'note_id' } + note_id_convert_to_bigint = table_columns.find { |c| c.name == 'note_id_convert_to_bigint' } + + note_id_convert_to_bigint.sql_type == 'integer' && note_id.sql_type == 'bigint' + end +end diff --git a/db/schema_migrations/20230413041917 b/db/schema_migrations/20230413041917 new file mode 100644 index 00000000000..f4c1f05a565 --- /dev/null +++ b/db/schema_migrations/20230413041917 @@ -0,0 +1 @@ +c359118e94c992bd19d8f0bd10f702bc117c944663a1f84f3d678bada242c09d
\ No newline at end of file diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 74598ff19a5..4a523ad75ec 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -376,6 +376,7 @@ configuration option in `gitlab.yml`. These metrics are served from the | `gitlab_memwd_violations_total` | Counter | 15.9 | Total number of times a Sidekiq process violated a memory threshold | | | `gitlab_memwd_violations_handled_total` | Counter | 15.9 | Total number of times Sidekiq process memory violations were handled | | | `sidekiq_watchdog_running_jobs_total` | Counter | 15.9 | Current running jobs when RSS limit was reached | `worker_class` | +| `gitlab_maintenance_mode` | Gauge | 15.11 | Is GitLab Maintenance Mode enabled? | | ## Database load balancing metrics **(PREMIUM SELF)** diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index bd9e7035290..fb5f6d03f13 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -22,6 +22,14 @@ Distributed tracing adds minimal overhead when disabled, but imposes only small enabled and is therefore capable in any environment, including production. For this reason, it can be useful in diagnosing production issues, particularly performance problems. +Services have different levels of support for distributed tracing. Custom +instrumentation code must be added to the application layer in addition to +pre-built instrumentation for the most common libraries. + +For service-specific information, see: + +- [Using Jaeger for Gitaly local development](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/jaeger_for_local_development.md) + ## Using Correlation IDs to investigate distributed requests The GitLab application passes correlation IDs between the various components in a request. A @@ -80,12 +88,41 @@ In this example, we have the following hypothetical values: ## Using Jaeger in the GitLab Development Kit The first tracing implementation that GitLab supports is Jaeger, and the -[GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit/) supports distributed tracing with -Jaeger out-of-the-box. +[GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit/) +supports distributed tracing with Jaeger out-of-the-box. GDK automatically adds +`GITLAB_TRACING` environment variables to add services. + +Configure GDK for Jaeger by editing the `gdk.yml` file and adding the following +settings: + +```yaml +tracer: + build_tags: tracer_static tracer_static_jaeger + jaeger: + enabled: true + listen_address: 127.0.0.1 + version: 1.43.0 +``` -The easiest way to access tracing from a GDK environment is through the -[performance-bar](../administration/monitoring/performance/performance_bar.md). This can be shown -by typing `p` `b` in the browser window. +After modifying the `gdk.yml` file, reconfigure your GDK by running +the `gdk reconfigure` command. This ensures that your GDK is properly configured +and ready to use. + +The above configuration sets the `tracer_static` and `tracer_static_jaeger` +build tags when rebuilding services written in Go for the first time. Any +changes made afterward require rebuilding them with those build tags. You can +either: + +- Add those build tags to the default set of build tags. +- Manually attach them to the build command. For example, Gitaly supports adding + build tag out of the box. You can run + `make all WITH_BUNDLED_GIT=YesPlease BUILD_TAGS="tracer_static tracer_static_jaeger"`. + +After reconfiguration, Jaeger dashboard is available at +`http://localhost:16686`. Another way to access tracing from a GDK environment +is through the +[performance-bar](../administration/monitoring/performance/performance_bar.md). +This can be shown by typing `p` `b` in the browser window. Once the performance bar is enabled, select **Trace** in the performance bar to go to the Jaeger UI. diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md index 2d9ae768d33..86142efa679 100644 --- a/doc/development/github_importer.md +++ b/doc/development/github_importer.md @@ -70,7 +70,12 @@ don't need to perform this work in parallel. This worker imports all pull requests. For every pull request a job for the `Gitlab::GithubImport::ImportPullRequestWorker` worker is scheduled. -### 5. Stage::ImportPullRequestsMergedByWorker +### 5. Stage::ImportCollaboratorsWorker + +This worker imports all repository collaborators. For every collaborator, we schedule a job +for the `Gitlab::GithubImport::ImportCollaboratorWorker` worker. + +### 6. Stage::ImportPullRequestsMergedByWorker This worker imports the pull requests' _merged-by_ user information. The [_List pull requests_](https://docs.github.com/en/rest/pulls#list-pull-requests) @@ -79,21 +84,21 @@ individually to import this information. A `Gitlab::GithubImport::ImportPullRequestMergedByWorker` job is scheduled for each fetched pull request. -### 6. Stage::ImportPullRequestsReviewRequestsWorker +### 7. Stage::ImportPullRequestsReviewRequestsWorker This worker imports assigned reviewers of pull requests. For each pull request, this worker: - Fetches all assigned review requests. - Schedules a `Gitlab::GithubImport::PullRequests::ImportReviewRequestWorker` job for each fetched review request. -### 7. Stage::ImportPullRequestsReviewsWorker +### 8. Stage::ImportPullRequestsReviewsWorker This worker imports reviews of pull requests. For each pull request, this worker: - Fetches all the pages of reviews. - Schedules a `Gitlab::GithubImport::ImportPullRequestReviewWorker` job for each fetched review. -### 8. Stage::ImportIssuesAndDiffNotesWorker +### 9. Stage::ImportIssuesAndDiffNotesWorker This worker imports all issues and pull request comments. For every issue, we schedule a job for the `Gitlab::GithubImport::ImportIssueWorker` worker. For @@ -109,7 +114,7 @@ label links in the same worker removes the need for performing a separate crawl through the API data, reducing the number of API calls necessary to import a project. -### 9. Stage::ImportIssueEventsWorker +### 10. Stage::ImportIssueEventsWorker This worker imports all issues and pull request events. For every event, we schedule a job for the `Gitlab::GithubImport::ImportIssueEventWorker` worker. @@ -125,7 +130,7 @@ Therefore, both issues and pull requests have a common API for most related thin NOTE: This stage is optional and can consume significant extra import time (controlled by `Gitlab::GithubImport::Settings`). -### 10. Stage::ImportNotesWorker +### 11. Stage::ImportNotesWorker This worker imports regular comments for both issues and pull requests. For every comment, we schedule a job for the @@ -136,7 +141,7 @@ returns comments for both issues and pull requests. This means we have to wait for all issues and pull requests to be imported before we can import regular comments. -### 11. Stage::ImportAttachmentsWorker +### 12. Stage::ImportAttachmentsWorker This worker imports note attachments that are linked inside Markdown. For each entity with Markdown text in the project, we schedule a job of: @@ -155,7 +160,7 @@ Each job: NOTE: It's an optional stage that could consume significant extra import time (controlled by `Gitlab::GithubImport::Settings`). -### 12. Stage::ImportProtectedBranchesWorker +### 13. Stage::ImportProtectedBranchesWorker This worker imports protected branch rules. For every rule that exists on GitHub, we schedule a job of @@ -164,7 +169,7 @@ For every rule that exists on GitHub, we schedule a job of Each job compares the branch protection rules from GitHub and GitLab and applies the strictest of the rules to the branches in GitLab. -### 13. Stage::FinishImportWorker +### 14. Stage::FinishImportWorker This worker completes the import process by performing some housekeeping (such as flushing any caches) and by marking the import as completed. diff --git a/doc/development/prometheus_metrics.md b/doc/development/prometheus_metrics.md index 590ac547c2d..f72805e0fe9 100644 --- a/doc/development/prometheus_metrics.md +++ b/doc/development/prometheus_metrics.md @@ -78,3 +78,15 @@ For example, a histogram with 10 buckets and a label with 100 values would gener entries in the export endpoint. 1. Trigger the relevant page or code that records the new metric. 1. Check that the new metric appears at `/-/metrics`. + +For metrics that are not bounded to a specific context (`request`, `process`, `machine`, `namespace`, etc), +generate them from a cron-based Sidekiq job: + +- For Geo related metrics, check `Geo::MetricsUpdateService`. +- For other "global" / instance-wide metrics, check: `Metrics::GlobalMetricsUpdateService`. + +When exporting data from Sidekiq in an installation with more than one Sidekiq instance, +you are not guaranteed that the same exporter will always be queried. + +You can read more and understand the caveats in [issue 406583](https://gitlab.com/gitlab-org/gitlab/-/issues/406583), +where we also discuss a possible solution using a push-gateway. diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 7e99f6e4e8c..acdaef851f5 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -7,7 +7,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Import your project from GitHub to GitLab **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381902) in GitLab 15.8, GitLab no longer automatically creates namespaces or groups that don't exist. GitLab also no longer falls back to using the user's personal namespace if the namespace or group name is taken. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381902) in GitLab 15.8, GitLab no longer automatically creates namespaces or groups that don't exist. GitLab also no longer falls back to using the user's personal namespace if the namespace or group name is taken. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/388716) in GitLab 15.10, you no longer need to add any users to the parent group in GitLab to successfully import the **Require a pull request before merging - Allow specified actors to bypass required pull requests** branch protection rule. You can import your GitHub projects from either GitHub.com or GitHub Enterprise. Importing projects does not migrate or import any types of groups or organizations from GitHub to GitLab. @@ -52,9 +53,6 @@ To import projects from GitHub: are properly mapped to the same user in GitLab. GitHub Enterprise does not require this field to be populated so you may have to add it on existing accounts. -See also [Branch protection rules and project settings](#branch-protection-rules-and-project-settings) for additional -prerequisites for those imports. - Because of a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/383047), if you are using GitHub as an OmniAuth provider, ensure that the URL perimeter is specified in the [OmniAuth configuration](../../../integration/github.md#enable-github-oauth-in-gitlab). @@ -255,10 +253,7 @@ When they are imported, supported GitHub branch protection rules are mapped to e | **Require signed commits** for the project's default branch | **Reject unsigned commits** GitLab [push rule](../repository/push_rules.md#prevent-unintended-consequences) **(PREMIUM)** | [GitLab 15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/370949) | | **Allow force pushes - Everyone** | **Allowed to force push** [branch protection setting](../protected_branches.md#allow-force-push-on-a-protected-branch) | [GitLab 15.6](https://gitlab.com/gitlab-org/gitlab/-/issues/370943) | | **Require a pull request before merging - Require review from Code Owners** | **Require approval from code owners** [branch protection setting](../protected_branches.md#require-code-owner-approval-on-a-protected-branch) **(PREMIUM)** | [GitLab 15.6](https://gitlab.com/gitlab-org/gitlab/-/issues/376683) | -| **Require a pull request before merging - Allow specified actors to bypass required pull requests** (1) | List of users in the **Allowed to push** list of [branch protection settings](../protected_branches.md#configure-a-protected-branch) **(PREMIUM)**. Without a Premium license, the list of users that are allowed to push is limited to roles. | [GitLab 15.8](https://gitlab.com/gitlab-org/gitlab/-/issues/384939) | - -1. To successfully import the **Require a pull request before merging - Allow specified actors to bypass required pull requests** rule, you must add to the parent group in GitLab - the users that are allowed to bypass required pull requests before you begin importing. +| **Require a pull request before merging - Allow specified actors to bypass required pull requests** | List of users in the **Allowed to push** list of [branch protection settings](../protected_branches.md#configure-a-protected-branch) **(PREMIUM)**. Without a **Premium** subscription, the list of users that are allowed to push is limited to roles. | [GitLab 15.8](https://gitlab.com/gitlab-org/gitlab/-/issues/384939) | Mapping GitHub rule **Require status checks to pass before merging** to [external status checks](../merge_requests/status_checks.md) was considered in issue diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb index 88896970bc6..8ddb2bb8815 100644 --- a/lib/banzai/reference_parser/commit_parser.rb +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -5,6 +5,8 @@ module Banzai class CommitParser < BaseParser self.reference_type = :commit + COMMITS_LIMIT = 1000 + def referenced_by(nodes, options = {}) commit_ids = commit_ids_per_project(nodes) projects = find_projects_for_hash_keys(commit_ids) @@ -19,6 +21,8 @@ module Banzai end def find_commits(project, ids) + return limited_commits(project, ids) if Feature.enabled?(:limited_commit_parser, project) + commits = [] return commits unless project.valid_repo? @@ -34,6 +38,14 @@ module Banzai private + def limited_commits(project, ids) + return [] unless project.valid_repo? + + ids = ids.take(COMMITS_LIMIT) + + project.commits_by(oids: ids) + end + def can_read_reference?(user, ref_project, node) can?(user, :download_code, ref_project) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index dbd24684208..f77169f6d2b 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -51,6 +51,11 @@ module Gitlab FULLY_QUALIFIED_IDENTIFIER = /^\w+\.\w+$/ + ## Database Modes + MODE_SINGLE_DATABASE = "single-database" + MODE_SINGLE_DATABASE_CI_CONNECTION = "single-database-ci-connection" + MODE_MULTIPLE_DATABASES = "multiple-databases" + def self.database_base_models @database_base_models ||= { # Note that we use ActiveRecord::Base here and not ApplicationRecord. @@ -138,7 +143,17 @@ module Gitlab # Database configured. Returns false if the database is shared def self.has_database?(database_name) db_config = ::Gitlab::Database.database_base_models[database_name]&.connection_db_config - db_config.present? && ::Gitlab::Database.db_config_share_with(db_config).nil? + db_config.present? && db_config_share_with(db_config).nil? + end + + def self.database_mode + if !has_config?(CI_DATABASE_NAME) + MODE_SINGLE_DATABASE + elsif has_database?(CI_DATABASE_NAME) + MODE_MULTIPLE_DATABASES + else + MODE_SINGLE_DATABASE_CI_CONNECTION + end end class PgUser < ApplicationRecord diff --git a/lib/gitlab/usage/metrics/instrumentations/database_mode.rb b/lib/gitlab/usage/metrics/instrumentations/database_mode.rb new file mode 100644 index 00000000000..1b97ef4a1d2 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/database_mode.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DatabaseMode < GenericMetric + value do + Gitlab::Database.database_mode + end + end + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 72013a5caaf..16ed2c0ae39 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -46754,6 +46754,9 @@ msgstr "" msgid "Unable to load commits. Try again later." msgstr "" +msgid "Unable to load current Workspaces. Please try again or contact an administrator." +msgstr "" + msgid "Unable to load file contents. Try again later." msgstr "" @@ -50106,6 +50109,9 @@ msgstr "" msgid "Workspaces|GitLab Workspaces is a powerful collaborative platform that provides a comprehensive set of tools for software development teams to manage their entire development lifecycle." msgstr "" +msgid "Workspaces|Workspaces" +msgstr "" + msgid "Would you like to create a new branch?" msgstr "" diff --git a/spec/lib/banzai/reference_parser/commit_parser_spec.rb b/spec/lib/banzai/reference_parser/commit_parser_spec.rb index 081bfa26fb2..9ed6235b8dd 100644 --- a/spec/lib/banzai/reference_parser/commit_parser_spec.rb +++ b/spec/lib/banzai/reference_parser/commit_parser_spec.rb @@ -5,8 +5,9 @@ require 'spec_helper' RSpec.describe Banzai::ReferenceParser::CommitParser, feature_category: :source_code_management do include ReferenceParserHelpers - let(:project) { create(:project, :public) } - let(:user) { create(:user) } + let_it_be(:project) { create(:project, :public, :repository) } + let_it_be(:user) { create(:user) } + subject { described_class.new(Banzai::RenderContext.new(project, user)) } let(:link) { empty_html_link } @@ -130,20 +131,50 @@ RSpec.describe Banzai::ReferenceParser::CommitParser, feature_category: :source_ end describe '#find_commits' do - it 'returns an Array of commit objects' do - commit = double(:commit) + let_it_be(:ids) { project.repository.commits(project.default_branch, limit: 3).map(&:id) } + + it 'is empty when repo is invalid' do + allow(project).to receive(:valid_repo?).and_return(false) - expect(project).to receive(:commit).with('123').and_return(commit) - expect(project).to receive(:valid_repo?).and_return(true) + expect(subject.find_commits(project, ids)).to eq([]) + end - expect(subject.find_commits(project, %w{123})).to eq([commit]) + it 'returns commits by the specified ids' do + expect(subject.find_commits(project, ids).map(&:id)).to eq(%w[ + b83d6e391c22777fca1ed3012fce84f633d7fed0 + 498214de67004b1da3d820901307bed2a68a8ef6 + 1b12f15a11fc6e62177bef08f47bc7b5ce50b141 + ]) end - it 'skips commit IDs for which no commit could be found' do - expect(project).to receive(:commit).with('123').and_return(nil) - expect(project).to receive(:valid_repo?).and_return(true) + it 'is limited' do + stub_const("#{described_class}::COMMITS_LIMIT", 1) + + expect(subject.find_commits(project, ids).map(&:id)).to eq([ + "b83d6e391c22777fca1ed3012fce84f633d7fed0" + ]) + end + + context 'when limited_commit_parser feature flag disabled' do + before do + stub_feature_flags(limited_commit_parser: false) + end + + it 'returns an Array of commit objects' do + commit = double(:commit) - expect(subject.find_commits(project, %w{123})).to eq([]) + expect(project).to receive(:commit).with('123').and_return(commit) + expect(project).to receive(:valid_repo?).and_return(true) + + expect(subject.find_commits(project, %w{123})).to eq([commit]) + end + + it 'skips commit IDs for which no commit could be found' do + expect(project).to receive(:commit).with('123').and_return(nil) + expect(project).to receive(:valid_repo?).and_return(true) + + expect(subject.find_commits(project, %w{123})).to eq([]) + end end end diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index 75c4b2fab4e..f2be888e6eb 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -85,6 +85,29 @@ RSpec.describe Gitlab::Database, feature_category: :database do end end + describe '.database_mode' do + context 'three tier database config' do + it 'returns single-database if ci is not configured' do + skip_if_multiple_databases_are_setup(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_SINGLE_DATABASE) + end + + it 'returns single-database-ci-connection if ci is shared with main database' do + skip_if_multiple_databases_not_setup(:ci) + skip_if_database_exists(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_SINGLE_DATABASE_CI_CONNECTION) + end + + it 'returns multiple-database if ci has its own database' do + skip_if_shared_database(:ci) + + expect(described_class.database_mode).to eq(::Gitlab::Database::MODE_MULTIPLE_DATABASES) + end + end + end + describe '.check_for_non_superuser' do subject { described_class.check_for_non_superuser } diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb new file mode 100644 index 00000000000..a6128b4df1f --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/database_mode_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMode, feature_category: :cell do + let(:expected_value) { Gitlab::Database.database_mode } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'none' } +end diff --git a/spec/migrations/swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com_spec.rb b/spec/migrations/swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com_2_spec.rb index 467f09ba8bd..2c561730d95 100644 --- a/spec/migrations/swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com_spec.rb +++ b/spec/migrations/swap_issue_user_mentions_note_id_to_bigint_for_gitlab_dot_com_2_spec.rb @@ -3,7 +3,8 @@ require 'spec_helper' require_migration! -RSpec.describe SwapIssueUserMentionsNoteIdToBigintForGitlabDotCom, feature_category: :database do +# rubocop: disable RSpec/FilePath +RSpec.describe SwapIssueUserMentionsNoteIdToBigintForGitlabDotCom2, feature_category: :database do describe '#up' do before do # A we call `schema_migrate_down!` before each example, and for this migration @@ -61,6 +62,23 @@ RSpec.describe SwapIssueUserMentionsNoteIdToBigintForGitlabDotCom, feature_categ end end end + + it 'is a no-op if columns are already swapped' do + connection = described_class.new.connection + connection.execute('ALTER TABLE issue_user_mentions ALTER COLUMN note_id TYPE bigint') + connection.execute('ALTER TABLE issue_user_mentions ALTER COLUMN note_id_convert_to_bigint TYPE integer') + + allow_any_instance_of(described_class).to receive(:com_or_dev_or_test_but_not_jh?).and_return(true) + + migrate! + + user_mentions = table(:issue_user_mentions) + user_mentions.reset_column_information + + expect(user_mentions.columns.find { |c| c.name == 'note_id' }.sql_type).to eq('bigint') + expect(user_mentions.columns.find { |c| c.name == 'note_id_convert_to_bigint' }.sql_type).to eq('integer') + end # rubocop: enable RSpec/AnyInstanceOf end end +# rubocop: enable RSpec/FilePath diff --git a/spec/services/metrics/global_metrics_update_service_spec.rb b/spec/services/metrics/global_metrics_update_service_spec.rb new file mode 100644 index 00000000000..38c7f9282d9 --- /dev/null +++ b/spec/services/metrics/global_metrics_update_service_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Metrics::GlobalMetricsUpdateService, :prometheus, feature_category: :metrics do + describe '#execute' do + it 'sets gitlab_maintenance_mode gauge metric' do + metric = subject.maintenance_mode_metric + expect(Gitlab).to receive(:maintenance_mode?).and_return(true) + + expect { subject.execute }.to change { metric.get }.from(0).to(1) + end + end +end diff --git a/spec/workers/metrics/global_metrics_update_worker_spec.rb b/spec/workers/metrics/global_metrics_update_worker_spec.rb new file mode 100644 index 00000000000..d5bfbcc928a --- /dev/null +++ b/spec/workers/metrics/global_metrics_update_worker_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ::Metrics::GlobalMetricsUpdateWorker, feature_category: :metrics do + subject { described_class.new } + + describe '#perform' do + let(:service) { ::Metrics::GlobalMetricsUpdateService.new } + + it 'delegates to ::Metrics::GlobalMetricsUpdateService' do + expect(::Metrics::GlobalMetricsUpdateService).to receive(:new).and_return(service) + expect(service).to receive(:execute) + + subject.perform + end + + context 'for an idempotent worker' do + include_examples 'an idempotent worker' do + it 'exports metrics' do + allow(Gitlab).to receive(:maintenance_mode?).and_return(true).at_least(1).time + + perform_multiple + + expect(service.maintenance_mode_metric.get).to eq(1) + end + end + end + end +end diff --git a/workhorse/go.mod b/workhorse/go.mod index a9d5e533653..87d5a311211 100644 --- a/workhorse/go.mod +++ b/workhorse/go.mod @@ -26,7 +26,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/smartystreets/goconvey v1.7.2 github.com/stretchr/testify v1.8.2 - gitlab.com/gitlab-org/gitaly/v15 v15.10.1 + gitlab.com/gitlab-org/gitaly/v15 v15.10.2 gitlab.com/gitlab-org/golang-archive-zip v0.1.1 gitlab.com/gitlab-org/labkit v1.18.0 gocloud.dev v0.29.0 diff --git a/workhorse/go.sum b/workhorse/go.sum index 45447c67e2c..7ca311b0a6d 100644 --- a/workhorse/go.sum +++ b/workhorse/go.sum @@ -1918,8 +1918,8 @@ github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -gitlab.com/gitlab-org/gitaly/v15 v15.10.1 h1:43RZwVktIZLOq2GsRvQVUWZ4YiKPEueoWnmzz3T/UWk= -gitlab.com/gitlab-org/gitaly/v15 v15.10.1/go.mod h1:ZsyTd8zGxT4WuSZJ0G8ydJb60mauzEP0XJtZJEy/7bc= +gitlab.com/gitlab-org/gitaly/v15 v15.10.2 h1:e3uqhqYAsl20VddIBpGngY3t6+nK6QnZjY7MhufhV2k= +gitlab.com/gitlab-org/gitaly/v15 v15.10.2/go.mod h1:ZsyTd8zGxT4WuSZJ0G8ydJb60mauzEP0XJtZJEy/7bc= gitlab.com/gitlab-org/golang-archive-zip v0.1.1 h1:35k9giivbxwF03+8A05Cm8YoxoakU8FBCj5gysjCTCE= gitlab.com/gitlab-org/golang-archive-zip v0.1.1/go.mod h1:ZDtqpWPGPB9qBuZnZDrKQjIdJtkN7ZAoVwhT6H2o2kE= gitlab.com/gitlab-org/labkit v1.18.0 h1:uYCIqDt/5V1hLIecTR4UNc1sD2+xiYplyKeyfpNN26A= |