From edc0ed44a95a52b3b2d3e28cf572a4a6a9680ddc Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 7 Mar 2019 17:19:56 -0800 Subject: Batch insert CI rspec_profiling data Instead of inserting a row after each example to an external database, we save the CI profiling reports into the `rspec_profiling` directory and insert the data in one batch in the update-tests-metadata CI stage. This should make each spec run faster and also reduce the number of PostgreSQL connections needed by concurrent CI builds. Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/10154 --- scripts/insert-rspec-profiling-data | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 scripts/insert-rspec-profiling-data (limited to 'scripts/insert-rspec-profiling-data') diff --git a/scripts/insert-rspec-profiling-data b/scripts/insert-rspec-profiling-data new file mode 100755 index 00000000000..e1fb042d098 --- /dev/null +++ b/scripts/insert-rspec-profiling-data @@ -0,0 +1,50 @@ +#!/usr/bin/env ruby + +require 'csv' +require 'rspec_profiling' + +module RspecProfiling + module Collectors + class PSQL + def establish_connection + Result.establish_connection(results_url) + end + + def prepared? + connection.data_source_exists?(table) + end + + def results_url + ENV['RSPEC_PROFILING_POSTGRES_URL'] + end + end + end +end + +def insert_data(path) + puts "#{Time.now} Inserting CI stats..." + + collector = RspecProfiling::Collectors::PSQL.new + collector.install + + files = Dir[File.join(path, "/*.csv")] + + puts "#{Time.now} Detected #{files.count} files in rspec_profiling" + + files.each do |filename| + csv_data = CSV.open(filename, headers: :first_row) + count = 0 + + puts "#{Time.now}: Inserting #{filename}..." + + csv_data.each do |entry| + entry = entry.to_h.each_value { |x| x&.strip! } + collector.insert(entry) + count += 1 + end + + puts "#{Time.now}: Done inserting #{count} lines in #{filename}" + end +end + +insert_data('rspec_profiling') if ENV['RSPEC_PROFILING_POSTGRES_URL'].present? -- cgit v1.2.1