diff options
| -rw-r--r-- | CHANGELOG | 4 | ||||
| -rw-r--r-- | app/models/ci/pipeline.rb | 13 | ||||
| -rw-r--r-- | doc/development/migration_style_guide.md | 1 | ||||
| -rw-r--r-- | doc/monitoring/performance/grafana_configuration.md | 86 | ||||
| -rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/metrics/sidekiq_middleware.rb | 2 | ||||
| -rw-r--r-- | spec/lib/gitlab/import_export/project.json | 23 | ||||
| -rw-r--r-- | spec/lib/gitlab/import_export/project_tree_restorer_spec.rb | 6 | ||||
| -rw-r--r-- | spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb | 17 |
9 files changed, 85 insertions, 71 deletions
diff --git a/CHANGELOG b/CHANGELOG index 76d9712ce9f..400414b1c26 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,9 +2,13 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.10.0 (unreleased) - Wrap code blocks on Activies and Todos page. !4783 (winniehell) + - Add Sidekiq queue duration to transaction metrics. - Fix MR-auto-close text added to description. !4836 - Implement Subresource Integrity for CSS and JavaScript assets. This prevents malicious assets from loading in the case of a CDN compromise. +v 8.9.1 + - Fix GitLab project import issues related to notes and builds + v 8.9.0 - Fix builds API response not including commit data - Fix error when CI job variables key specified but not defined diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ca5a685dd11..0c9a5e42eec 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -170,6 +170,19 @@ module Ci builds.where.not(environment: nil).success.pluck(:environment).uniq end + # Manually set the notes for a Ci::Pipeline + # There is no ActiveRecord relation between Ci::Pipeline and notes + # as they are related to a commit sha. This method helps importing + # them using the +Gitlab::ImportExport::RelationFactory+ class. + def notes=(notes) + notes.each do |note| + note[:id] = nil + note[:commit_id] = sha + note[:noteable_id] = self['id'] + note.save! + end + end + def notes Note.for_commit_id(sha) end diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 8a7547e5322..e2ca46504e7 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -37,7 +37,6 @@ First, you need to provide information on whether the migration can be applied: For example: ``` -# rubocop:disable all # Migration type: online without errors (works on previous version and new one) class MyMigration < ActiveRecord::Migration ... diff --git a/doc/monitoring/performance/grafana_configuration.md b/doc/monitoring/performance/grafana_configuration.md index b5a3d33129b..7947b0fedc4 100644 --- a/doc/monitoring/performance/grafana_configuration.md +++ b/doc/monitoring/performance/grafana_configuration.md @@ -44,82 +44,32 @@ on a separate server) ## Apply retention policies and create continuous queries -If you intend to import the GitLab provided Grafana dashboards, you will need -to copy and run a set of queries against InfluxDB to create the needed data -sets. +If you intend to import the GitLab provided Grafana dashboards, you will need to +set up the right retention policies and continuous queries. The easiest way of +doing this is by using the [influxdb-management](https://gitlab.com/gitlab-org/influxdb-management) +repository. -On the InfluxDB server, run the following command, substituting your InfluxDB -user and password: +To use this repository you must first clone it: -```bash -influxdb --username admin -password super_secret +``` +git clone https://gitlab.com/gitlab-org/influxdb-management.git +cd influxdb-management ``` -This will drop you in to an InfluxDB interactive session. Copy the entire -contents below and paste it in to the interactive session: +Next you must install the required dependencies: ``` -CREATE RETENTION POLICY default ON gitlab DURATION 1h REPLICATION 1 DEFAULT -CREATE RETENTION POLICY downsampled ON gitlab DURATION 7d REPLICATION 1 -CREATE CONTINUOUS QUERY grape_git_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO downsampled.grape_git_timings_per_action FROM "default".rails_method_calls WHERE (action !~ /.+/ OR action =~ /^Grape#/) AND method =~ /^(Rugged|Gitlab::Git)/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY grape_markdown_render_timings_overall ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO downsampled.grape_markdown_render_timings_overall FROM "default".rails_transactions WHERE (action !~ /.+/ OR action =~ /^Grape#/) AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m) END -CREATE CONTINUOUS QUERY grape_markdown_render_timings_per_action ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO downsampled.grape_markdown_render_timings_per_action FROM "default".rails_transactions WHERE (action !~ /.+/ OR action =~ /^Grape#/) AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY grape_markdown_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO downsampled.grape_markdown_timings_overall FROM "default".rails_method_calls WHERE (action !~ /.+/ OR action =~ /^Grape#/) AND method =~ /^Banzai/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY grape_method_call_timings_per_action_and_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(cpu_duration) AS cpu_duration_mean, percentile(cpu_duration, 95) AS cpu_duration_95th, percentile(cpu_duration, 99) AS cpu_duration_99th, mean(call_count) AS call_count_mean, percentile(call_count, 95) AS call_count_95th, percentile(call_count, 99) AS call_count_99th INTO downsampled.grape_method_call_timings_per_action_and_method FROM "default".rails_method_calls WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), method, action END -CREATE CONTINUOUS QUERY grape_method_call_timings_per_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(cpu_duration) AS cpu_duration_mean, percentile(cpu_duration, 95) AS cpu_duration_95th, percentile(cpu_duration, 99) AS cpu_duration_99th, mean(call_count) AS call_count_mean, percentile(call_count, 95) AS call_count_95th, percentile(call_count, 99) AS call_count_99th INTO downsampled.grape_method_call_timings_per_method FROM "default".rails_method_calls WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), method END -CREATE CONTINUOUS QUERY grape_transaction_counts_overall ON gitlab BEGIN SELECT count("duration") AS count INTO downsampled.grape_transaction_counts_overall FROM "default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY grape_transaction_counts_per_action ON gitlab BEGIN SELECT count("duration") AS count INTO downsampled.grape_transaction_counts_per_action FROM "default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY grape_transaction_new_redis_connection_counts_overall ON gitlab BEGIN SELECT max(new_redis_connections) AS new_redis_connections_max, mean(new_redis_connections) AS new_redis_connections_mean, percentile(new_redis_connections, 95) AS new_redis_connections_95th, percentile(new_redis_connections, 99) AS new_redis_connections_99th INTO downsampled.grape_transaction_new_redis_connection_counts_overall FROM "default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY grape_transaction_new_redis_connection_counts_per_action ON gitlab BEGIN SELECT max(new_redis_connections) AS new_redis_connections_max, mean(new_redis_connections) AS new_redis_connections_mean, percentile(new_redis_connections, 95) AS new_redis_connections_95th, percentile(new_redis_connections, 99) AS new_redis_connections_99th INTO downsampled.grape_transaction_new_redis_connection_counts_per_action FROM "default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY grape_transaction_sql_counts_overall ON gitlab BEGIN SELECT max(sql_count) AS sql_count_max, mean(sql_count) AS sql_count_mean, percentile(sql_count, 95) AS sql_count_95th, percentile(sql_count, 99) AS sql_count_99th INTO downsampled.grape_transaction_sql_counts_overall FROM "default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY grape_transaction_sql_counts_per_action ON gitlab BEGIN SELECT max(sql_count) AS sql_count_max, mean(sql_count) AS sql_count_mean, percentile(sql_count, 95) AS sql_count_95th, percentile(sql_count, 99) AS sql_count_99th INTO downsampled.grape_transaction_sql_counts_per_action FROM "default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY grape_transaction_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, max(sql_duration) AS sql_duration_max, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(rails_queue_duration) AS rails_queue_duration_mean, percentile(rails_queue_duration, 95) AS rails_queue_duration_95th, percentile(rails_queue_duration, 99) AS rails_queue_duration_99th INTO downsampled.grape_transaction_timings_overall FROM "default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY grape_transaction_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, max(sql_duration) AS sql_duration_max, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(rails_queue_duration) AS rails_queue_duration_mean, percentile(rails_queue_duration, 95) AS rails_queue_duration_95th, percentile(rails_queue_duration, 99) AS rails_queue_duration_99th INTO downsampled.grape_transaction_timings_per_action FROM "default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY rails_file_descriptor_counts ON gitlab BEGIN SELECT sum(value) AS count INTO downsampled.rails_file_descriptor_counts FROM "default".rails_file_descriptors GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_gc_counts ON gitlab BEGIN SELECT sum(count) AS total, sum(minor_gc_count) AS minor, sum(major_gc_count) AS major INTO downsampled.rails_gc_counts FROM "default".rails_gc_statistics GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_gc_timings ON gitlab BEGIN SELECT mean(total_time) AS duration_mean, percentile(total_time, 95) AS duration_95th, percentile(total_time, 99) AS duration_99th INTO downsampled.rails_gc_timings FROM "default".rails_gc_statistics GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_git_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO downsampled.rails_git_timings_per_action FROM "default".rails_method_calls WHERE (action =~ /.+/ AND action !~ /^Grape#/) AND method =~ /^(Rugged|Gitlab::Git)/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY rails_markdown_render_timings_overall ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO downsampled.rails_markdown_render_timings_overall FROM "default".rails_transactions WHERE (action =~ /.+/ AND action !~ /^Grape#/) AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_markdown_render_timings_per_action ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO downsampled.rails_markdown_render_timings_per_action FROM "default".rails_transactions WHERE (action =~ /.+/ AND action !~ /^Grape#/) AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY rails_markdown_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO downsampled.rails_markdown_timings_overall FROM "default".rails_method_calls WHERE (action =~ /.+/ AND action !~ /^Grape#/) AND method =~ /^Banzai/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_memory_usage_overall ON gitlab BEGIN SELECT mean(value) AS memory_mean, percentile(value, 95) AS memory_95th, percentile(value, 99) AS memory_99th INTO downsampled.rails_memory_usage_overall FROM "default".rails_memory_usage GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_method_call_timings_per_action_and_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(cpu_duration) AS cpu_duration_mean, percentile(cpu_duration, 95) AS cpu_duration_95th, percentile(cpu_duration, 99) AS cpu_duration_99th, mean(call_count) AS call_count_mean, percentile(call_count, 95) AS call_count_95th, percentile(call_count, 99) AS call_count_99th INTO downsampled.rails_method_call_timings_per_action_and_method FROM "default".rails_method_calls WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), method, action END -CREATE CONTINUOUS QUERY rails_method_call_timings_per_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(cpu_duration) AS cpu_duration_mean, percentile(cpu_duration, 95) AS cpu_duration_95th, percentile(cpu_duration, 99) AS cpu_duration_99th, mean(call_count) AS call_count_mean, percentile(call_count, 95) AS call_count_95th, percentile(call_count, 99) AS call_count_99th INTO downsampled.rails_method_call_timings_per_method FROM "default".rails_method_calls WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), method END -CREATE CONTINUOUS QUERY rails_object_counts_overall ON gitlab BEGIN SELECT sum(count) AS count INTO downsampled.rails_object_counts_overall FROM "default".rails_object_counts GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_object_counts_per_type ON gitlab BEGIN SELECT sum(count) AS count INTO downsampled.rails_object_counts_per_type FROM "default".rails_object_counts GROUP BY time(1m), type END -CREATE CONTINUOUS QUERY rails_transaction_counts_overall ON gitlab BEGIN SELECT count("duration") AS count INTO downsampled.rails_transaction_counts_overall FROM "default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_transaction_counts_per_action ON gitlab BEGIN SELECT count("duration") AS count INTO downsampled.rails_transaction_counts_per_action FROM "default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY rails_transaction_new_redis_connection_counts_overall ON gitlab BEGIN SELECT max(new_redis_connections) AS new_redis_connections_max, mean(new_redis_connections) AS new_redis_connections_mean, percentile(new_redis_connections, 95) AS new_redis_connections_95th, percentile(new_redis_connections, 99) AS new_redis_connections_99th INTO downsampled.rails_transaction_new_redis_connection_counts_overall FROM "default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_transaction_new_redis_connection_counts_per_action ON gitlab BEGIN SELECT max(new_redis_connections) AS new_redis_connections_max, mean(new_redis_connections) AS new_redis_connections_mean, percentile(new_redis_connections, 95) AS new_redis_connections_95th, percentile(new_redis_connections, 99) AS new_redis_connections_99th INTO downsampled.rails_transaction_new_redis_connection_counts_per_action FROM "default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY rails_transaction_sql_counts_overall ON gitlab BEGIN SELECT max(sql_count) AS sql_count_max, mean(sql_count) AS sql_count_mean, percentile(sql_count, 95) AS sql_count_95th, percentile(sql_count, 99) AS sql_count_99th INTO downsampled.rails_transaction_sql_counts_overall FROM "default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_transaction_sql_counts_per_action ON gitlab BEGIN SELECT max(sql_count) AS sql_count_max, mean(sql_count) AS sql_count_mean, percentile(sql_count, 95) AS sql_count_95th, percentile(sql_count, 99) AS sql_count_99th INTO downsampled.rails_transaction_sql_counts_per_action FROM "default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY rails_transaction_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, max(sql_duration) AS sql_duration_max, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(rails_queue_duration) AS rails_queue_duration_mean, percentile(rails_queue_duration, 95) AS rails_queue_duration_95th, percentile(rails_queue_duration, 99) AS rails_queue_duration_99th INTO downsampled.rails_transaction_timings_overall FROM "default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY rails_transaction_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, max(sql_duration) AS sql_duration_max, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(rails_queue_duration) AS rails_queue_duration_mean, percentile(rails_queue_duration, 95) AS rails_queue_duration_95th, percentile(rails_queue_duration, 99) AS rails_queue_duration_99th INTO downsampled.rails_transaction_timings_per_action FROM "default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY rails_view_timings_per_action_and_view ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO downsampled.rails_view_timings_per_action_and_view FROM "default".rails_views WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), action, view END -CREATE CONTINUOUS QUERY sidekiq_file_descriptor_counts ON gitlab BEGIN SELECT sum(value) AS count INTO downsampled.sidekiq_file_descriptor_counts FROM "default".sidekiq_file_descriptors GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_gc_counts ON gitlab BEGIN SELECT sum(count) AS total, sum(minor_gc_count) AS minor, sum(major_gc_count) AS major INTO downsampled.sidekiq_gc_counts FROM "default".sidekiq_gc_statistics GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_gc_timings ON gitlab BEGIN SELECT mean(total_time) AS duration_mean, percentile(total_time, 95) AS duration_95th, percentile(total_time, 99) AS duration_99th INTO downsampled.sidekiq_gc_timings FROM "default".sidekiq_gc_statistics GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_git_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO downsampled.sidekiq_git_timings_per_action FROM "default".sidekiq_method_calls WHERE method =~ /^(Rugged|Gitlab::Git)/ GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY sidekiq_markdown_render_timings_overall ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO downsampled.sidekiq_markdown_render_timings_overall FROM "default".sidekiq_transactions WHERE (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_markdown_render_timings_per_action ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO downsampled.sidekiq_markdown_render_timings_per_action FROM "default".sidekiq_transactions WHERE (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY sidekiq_markdown_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO downsampled.sidekiq_markdown_timings_overall FROM "default".sidekiq_method_calls WHERE method =~ /^Banzai/ GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_memory_usage_overall ON gitlab BEGIN SELECT mean(value) AS memory_mean, percentile(value, 95) AS memory_95th, percentile(value, 99) AS memory_99th INTO downsampled.sidekiq_memory_usage_overall FROM "default".sidekiq_memory_usage GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_method_call_timings_per_action_and_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(cpu_duration) AS cpu_duration_mean, percentile(cpu_duration, 95) AS cpu_duration_95th, percentile(cpu_duration, 99) AS cpu_duration_99th, mean(call_count) AS call_count_mean, percentile(call_count, 95) AS call_count_95th, percentile(call_count, 99) AS call_count_99th INTO downsampled.sidekiq_method_call_timings_per_action_and_method FROM "default".sidekiq_method_calls GROUP BY time(1m), method, action END -CREATE CONTINUOUS QUERY sidekiq_method_call_timings_per_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(cpu_duration) AS cpu_duration_mean, percentile(cpu_duration, 95) AS cpu_duration_95th, percentile(cpu_duration, 99) AS cpu_duration_99th, mean(call_count) AS call_count_mean, percentile(call_count, 95) AS call_count_95th, percentile(call_count, 99) AS call_count_99th INTO downsampled.sidekiq_method_call_timings_per_method FROM "default".sidekiq_method_calls GROUP BY time(1m), method END -CREATE CONTINUOUS QUERY sidekiq_object_counts_overall ON gitlab BEGIN SELECT sum(count) AS count INTO downsampled.sidekiq_object_counts_overall FROM "default".sidekiq_object_counts GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_object_counts_per_type ON gitlab BEGIN SELECT sum(count) AS count INTO downsampled.sidekiq_object_counts_per_type FROM "default".sidekiq_object_counts GROUP BY time(1m), type END -CREATE CONTINUOUS QUERY sidekiq_transaction_counts_overall ON gitlab BEGIN SELECT count("duration") AS count INTO downsampled.sidekiq_transaction_counts_overall FROM "default".sidekiq_transactions GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_transaction_counts_per_action ON gitlab BEGIN SELECT count("duration") AS count INTO downsampled.sidekiq_transaction_counts_per_action FROM "default".sidekiq_transactions GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY sidekiq_transaction_new_redis_connection_counts_overall ON gitlab BEGIN SELECT max(new_redis_connections) AS new_redis_connections_max, mean(new_redis_connections) AS new_redis_connections_mean, percentile(new_redis_connections, 95) AS new_redis_connections_95th, percentile(new_redis_connections, 99) AS new_redis_connections_99th INTO downsampled.sidekiq_transaction_new_redis_connection_counts_overall FROM "default".sidekiq_transactions GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_transaction_new_redis_connection_counts_per_action ON gitlab BEGIN SELECT max(new_redis_connections) AS new_redis_connections_max, mean(new_redis_connections) AS new_redis_connections_mean, percentile(new_redis_connections, 95) AS new_redis_connections_95th, percentile(new_redis_connections, 99) AS new_redis_connections_99th INTO downsampled.sidekiq_transaction_new_redis_connection_counts_per_action FROM "default".sidekiq_transactions GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY sidekiq_transaction_sql_counts_overall ON gitlab BEGIN SELECT max(sql_count) AS sql_count_max, mean(sql_count) AS sql_count_mean, percentile(sql_count, 95) AS sql_count_95th, percentile(sql_count, 99) AS sql_count_99th INTO downsampled.sidekiq_transaction_sql_counts_overall FROM "default".sidekiq_transactions GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_transaction_sql_counts_per_action ON gitlab BEGIN SELECT max(sql_count) AS sql_count_max, mean(sql_count) AS sql_count_mean, percentile(sql_count, 95) AS sql_count_95th, percentile(sql_count, 99) AS sql_count_99th INTO downsampled.sidekiq_transaction_sql_counts_per_action FROM "default".sidekiq_transactions GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY sidekiq_transaction_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, max(sql_duration) AS sql_duration_max, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th INTO downsampled.sidekiq_transaction_timings_overall FROM "default".sidekiq_transactions GROUP BY time(1m) END -CREATE CONTINUOUS QUERY sidekiq_transaction_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, max(sql_duration) AS sql_duration_max, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th INTO downsampled.sidekiq_transaction_timings_per_action FROM "default".sidekiq_transactions GROUP BY time(1m), action END -CREATE CONTINUOUS QUERY sidekiq_view_timings_per_action_and_view ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO downsampled.sidekiq_view_timings_per_action_and_view FROM "default".sidekiq_views GROUP BY time(1m), action, view END -CREATE CONTINUOUS QUERY web_transaction_counts_overall ON gitlab BEGIN SELECT count("duration") AS count INTO downsampled.web_transaction_counts_overall FROM "default".rails_transactions GROUP BY time(1m) END +gem install bundler +bundle install ``` +Now you must configure the repository by first copying `.env.example` to `.env` +and then editing the `.env` file to contain the correct InfluxDB settings. Once +configured you can simply run `bundle exec rake` and the InfluxDB database will +be configured for you. + +For more information see the [influxdb-management README](https://gitlab.com/gitlab-org/influxdb-management/blob/master/README.md). + ## Import Dashboards You can now import a set of default dashboards that will give you a good diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index b872780f20a..92bf7e0a2fc 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -12,6 +12,8 @@ module Gitlab USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + BUILD_MODELS = %w[Ci::Build commit_status].freeze + def self.create(*args) new(*args).create end @@ -70,7 +72,7 @@ module Gitlab end def generate_imported_object - if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes trace = @relation_hash.delete('trace') imported_object do |object| object.trace = trace diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index fd98aa3412e..a1240fd33ee 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -8,6 +8,8 @@ module Gitlab trans = Transaction.new("#{worker.class.name}#perform") begin + # Old gitlad-shell messages don't provide enqueued_at/created_at attributes + trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } ensure trans.finish diff --git a/spec/lib/gitlab/import_export/project.json b/spec/lib/gitlab/import_export/project.json index 400d44ac162..403bd582ef3 100644 --- a/spec/lib/gitlab/import_export/project.json +++ b/spec/lib/gitlab/import_export/project.json @@ -4894,6 +4894,29 @@ "started_at": null, "finished_at": null, "duration": null, + "notes": [ + { + "id": 999, + "note": "Natus rerum qui dolorem dolorum voluptas.", + "noteable_type": "Commit", + "author_id": 1, + "created_at": "2016-03-22T15:19:59.469Z", + "updated_at": "2016-03-22T15:19:59.469Z", + "project_id": 5, + "attachment": { + "url": null + }, + "line_code": null, + "commit_id": "be93687618e4b132087f430a4d8fc3a609c9b77c", + "noteable_id": 36, + "system": false, + "st_diff": null, + "updated_by_id": null, + "author": { + "name": "Administrator" + } + } + ], "statuses": [ { "id": 71, diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb index 7a40a43f8ae..23036ab8108 100644 --- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb @@ -18,6 +18,12 @@ describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do it 'restores models based on JSON' do expect(restored_project_json).to be true end + + it 'creates a valid pipeline note' do + restored_project_json + + expect(Ci::Pipeline.first.notes).not_to be_empty + end end end end diff --git a/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb b/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb index e520a968999..4d2aa03e722 100644 --- a/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb +++ b/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb @@ -2,6 +2,7 @@ require 'spec_helper' describe Gitlab::Metrics::SidekiqMiddleware do let(:middleware) { described_class.new } + let(:message) { { 'args' => ['test'], 'enqueued_at' => Time.new(2016, 6, 23, 6, 59).to_f } } describe '#call' do it 'tracks the transaction' do @@ -11,9 +12,23 @@ describe Gitlab::Metrics::SidekiqMiddleware do with('TestWorker#perform'). and_call_original + expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set).with(:sidekiq_queue_duration, instance_of(Float)) expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:finish) - middleware.call(worker, 'test', :test) { nil } + middleware.call(worker, message, :test) { nil } + end + + it 'tracks the transaction (for messages without `enqueued_at`)' do + worker = double(:worker, class: double(:class, name: 'TestWorker')) + + expect(Gitlab::Metrics::Transaction).to receive(:new). + with('TestWorker#perform'). + and_call_original + + expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set).with(:sidekiq_queue_duration, instance_of(Float)) + expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:finish) + + middleware.call(worker, {}, :test) { nil } end end end |