From a1bee1a09d529007c4ff11ab55d355b51f352ed8 Mon Sep 17 00:00:00 2001 From: Desiree Chevalier Date: Fri, 7 Jun 2019 08:46:48 -0400 Subject: Generate parallel spec reports Creates reports for parallel specs and collates them into a single HTML report and displays results on the merge request. --- scripts/merge-html-reports | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 scripts/merge-html-reports (limited to 'scripts') diff --git a/scripts/merge-html-reports b/scripts/merge-html-reports new file mode 100755 index 00000000000..7d1e15186c8 --- /dev/null +++ b/scripts/merge-html-reports @@ -0,0 +1,84 @@ +#!/usr/bin/env ruby + +require 'nokogiri' + +main_report_file = ARGV.shift +unless main_report_file + puts 'usage: merge-html-reports [parallel reports...]' + exit 1 +end + +base_artifact_url = ARGV.shift +unless base_artifact_url + puts 'usage: merge-html-reports [parallel reports...]' + exit 1 +end + +# Create the base report with empty body tag +new_report = Nokogiri::HTML.parse(File.read(ARGV[0])) +new_report.at_css('body').remove +empty_body = Nokogiri::XML::Node.new('body', new_report) +new_report.at_css('head').add_next_sibling(empty_body) + +ARGV.each do |report_file| + report = Nokogiri::HTML.parse(File.read(report_file)) + + report.css('a').each do |link| + link_suffix = link['href'].slice(19..-1) + link['href'] = base_artifact_url + link_suffix + end + + header = report.css('div #rspec-header') + tests = report.css('dt[id^="example_group_"]') + + tests.each do |test| + title = test.parent + group = title.parent + script = title.css('script') + + if script.inner_html.include? 'makeYellow' + test.remove_class('passed') + test.add_class('not_implemented') + + group.remove_class('passed') + group.add_class('not_implemented') + header.add_class('not_implemented') + + script.remove + test.next_sibling.remove + test.next_sibling.remove + + elsif script.inner_html.include? 'makeRed' + test.remove_class('passed') + test.add_class('failed') + + group.remove_class('passed') + group.add_class('failed') + header.add_class('failed') + + script.remove + test.next_sibling.remove + test.next_sibling.remove + end + end + + duration = report.at_css('p#duration') + totals = report.at_css('p#totals') + + duration_script = report.css('div.results script')[-2] + totals_script = report.css('div.results script')[-1] + + duration_text = duration_script.text.slice(49..-3) + totals_text = totals_script.text.slice(47..-3) + + duration.inner_html = duration_text + totals.inner_html = totals_text + + duration_script.remove + totals_script.remove + + # Add the new result after the last one to keep the test order + new_report.css('body')[-1].add_next_sibling(report.at_css('body')) +end + +File.write(main_report_file, new_report) -- cgit v1.2.1 From 1136c0c8e98d4f0d3fb4f50219657cabe0d45c99 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 17 Jul 2019 16:34:27 -0700 Subject: Add Rugged calls and duration to API and Rails logs This adds `rugged_duration_ms` and `rugged_calls` fields to `api_json.log` and `production_json.log`. This will make it easier to identify performance issues caused by excessive I/O. Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/64676 --- scripts/lint-rugged | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lint-rugged b/scripts/lint-rugged index 9466c62a415..e1605b4501b 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -10,7 +10,12 @@ ALLOWED = [ # Reverted Rugged calls due to Gitaly atop NFS performance # See https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code. 'lib/gitlab/git/rugged_impl/', - 'lib/gitlab/gitaly_client/storage_settings.rb' + 'lib/gitlab/gitaly_client/storage_settings.rb', + + # Needed for logging + 'config/initializers/lograge.rb', + 'lib/gitlab/grape_logging/loggers/perf_logger.rb', + 'lib/gitlab/rugged_instrumentation.rb' ].freeze rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines -- cgit v1.2.1 From b2f143597ca22b9585cc758453ae920d9b9d0fe3 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Tue, 23 Jul 2019 12:23:51 +0000 Subject: Extend yamllint test to changelogs cc !30979 --- scripts/lint-changelog-yaml | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100755 scripts/lint-changelog-yaml (limited to 'scripts') diff --git a/scripts/lint-changelog-yaml b/scripts/lint-changelog-yaml deleted file mode 100755 index 06d502c4676..00000000000 --- a/scripts/lint-changelog-yaml +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env ruby - -require 'yaml' - -invalid_changelogs = Dir['changelogs/**/*'].reject do |changelog| - next true if changelog =~ /((README|archive)\.md|unreleased(-ee)?)$/ - next false unless changelog.end_with?('.yml') - - begin - YAML.load_file(changelog) - rescue => exception - puts exception - end -end - -if invalid_changelogs.any? - puts - puts "Invalid changelogs found!\n" - puts invalid_changelogs.sort - exit 1 -else - puts "All changelogs are valid YAML.\n" - exit 0 -end -- cgit v1.2.1 From dccf643c42f8887844ab6e88f21d161d66c893e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 23 Jul 2019 17:11:13 +0200 Subject: Fix Review App debugging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- scripts/review_apps/review-apps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index bc47884ee45..363ab2c6540 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -52,7 +52,7 @@ function delete() { function get_pod() { local app_name="${1}" local status="${2-Running}" - get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name" + get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1" echoinfo "Waiting till '${app_name}' pod is ready" true echoinfo "Running '${get_pod_cmd}'" -- cgit v1.2.1 From 4aa76dddecc048cef24963323afe59f1c120cb72 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 13 Jun 2019 14:12:28 +0100 Subject: Remove dead MySQL code None of this code can be reached any more, so it can all be removed --- scripts/create_mysql_user.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 scripts/create_mysql_user.sh (limited to 'scripts') diff --git a/scripts/create_mysql_user.sh b/scripts/create_mysql_user.sh deleted file mode 100644 index 35f68c581f3..00000000000 --- a/scripts/create_mysql_user.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -mysql --user=root --host=mysql < Date: Tue, 23 Jul 2019 17:53:17 +0200 Subject: Increase the Review App deploy timeout to 15 minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- scripts/review_apps/review-apps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index bc47884ee45..2b565347f86 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -201,7 +201,7 @@ function deploy() { HELM_CMD=$(cat << EOF helm upgrade --install \ --wait \ - --timeout 600 \ + --timeout 900 \ --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ --set global.appConfig.enableUsagePing=false \ --set global.imagePullPolicy=Always \ -- cgit v1.2.1 From 28d9e87d408da31eae1b7284944083323afa600a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 23 Jul 2019 18:30:47 +0200 Subject: Stop printing dots while retrieving pod in review-apps.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- scripts/review_apps/review-apps.sh | 2 -- 1 file changed, 2 deletions(-) (limited to 'scripts') diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index bc47884ee45..ab65a13923f 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -69,7 +69,6 @@ function get_pod() { break fi - printf "." let "elapsed_seconds+=interval" sleep ${interval} done @@ -360,7 +359,6 @@ function wait_for_review_app_to_be_accessible() { break fi - printf "." let "elapsed_seconds+=interval" sleep ${interval} done -- cgit v1.2.1 From 291df05e434f5678c47bce9521ff15748d6c767f Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 20 Jul 2019 22:34:46 -0700 Subject: Add Rugged calls to performance bar This will help diagnose the source of excessive I/O from Rugged calls. To implement this, we need to obtain the full list of arguments sent to each request method. --- scripts/lint-rugged | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lint-rugged b/scripts/lint-rugged index e1605b4501b..f40f0489c1a 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -13,9 +13,11 @@ ALLOWED = [ 'lib/gitlab/gitaly_client/storage_settings.rb', # Needed for logging + 'config/initializers/peek.rb', 'config/initializers/lograge.rb', 'lib/gitlab/grape_logging/loggers/perf_logger.rb', - 'lib/gitlab/rugged_instrumentation.rb' + 'lib/gitlab/rugged_instrumentation.rb', + 'lib/peek/views/rugged.rb' ].freeze rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines -- cgit v1.2.1 From 2b3d00a77822eaf2e622dd0b1baf85ebea2b1ee4 Mon Sep 17 00:00:00 2001 From: Heinrich Lee Yu Date: Wed, 17 Jul 2019 11:51:22 +0800 Subject: Remove unneeded monkey-patch Changes all calls to data_source_exists? to table_exists? since that is the intent of these calls --- scripts/insert-rspec-profiling-data | 4 ---- 1 file changed, 4 deletions(-) (limited to 'scripts') diff --git a/scripts/insert-rspec-profiling-data b/scripts/insert-rspec-profiling-data index b34379764e0..88c9d8c12b1 100755 --- a/scripts/insert-rspec-profiling-data +++ b/scripts/insert-rspec-profiling-data @@ -14,10 +14,6 @@ module RspecProfiling Result.establish_connection(results_url) end - def prepared? - connection.data_source_exists?(table) - end - def results_url ENV['RSPEC_PROFILING_POSTGRES_URL'] end -- cgit v1.2.1 From d7a4c1d2a5f395ca6ddb2dc2608ac33ccebf8a74 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 22 Jul 2019 17:11:50 +0200 Subject: Backport EE changes made to the scripts/ directory This backport changes made by EE to the files in the scripts/ directory. This comes with a few changes to some scripts to make them work in the single codebase setup. --- scripts/lint-rugged | 3 ++ scripts/static-analysis | 5 ++ scripts/utils.sh | 128 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+) (limited to 'scripts') diff --git a/scripts/lint-rugged b/scripts/lint-rugged index f40f0489c1a..d862571c1c5 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -1,6 +1,9 @@ #!/usr/bin/env ruby ALLOWED = [ + # https://gitlab.com/gitlab-org/gitaly/issues/760 + 'lib/elasticsearch/git/repository.rb', + # Needed to handle repositories that are not in any storage 'lib/gitlab/bare_repository_import/repository.rb', diff --git a/scripts/static-analysis b/scripts/static-analysis index 642c50ec0a8..6fd64fbf9da 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -1,6 +1,7 @@ #!/usr/bin/env ruby # We don't have auto-loading here +require_relative '../lib/gitlab' require_relative '../lib/gitlab/popen' require_relative '../lib/gitlab/popen/runner' @@ -36,6 +37,10 @@ tasks = [ %w[scripts/lint-rugged] ] +if Gitlab.ee? + tasks.unshift(%w[ruby -rbundler/setup scripts/ee_specific_check/ee_specific_check.rb]) +end + static_analysis = Gitlab::Popen::Runner.new static_analysis.run(tasks) do |cmd, &run| diff --git a/scripts/utils.sh b/scripts/utils.sh index 4a6567b8a62..c1bdef5b847 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -29,6 +29,134 @@ function setup_db() { if [ "$GITLAB_DATABASE" = "mysql" ]; then bundle exec rake add_limits_mysql fi + + bundle exec rake gitlab:db:setup_ee +} + +function install_api_client_dependencies_with_apk() { + apk add --update openssl curl jq +} + +function install_api_client_dependencies_with_apt() { + apt update && apt install jq -y +} + +function install_gitlab_gem() { + gem install gitlab --no-document +} + +function echoerr() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2; + else + printf "\033[0;31m%s\n\033[0m" "${1}" >&2; + fi +} + +function echoinfo() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2; + else + printf "\033[0;33m%s\n\033[0m" "${1}" >&2; + fi +} + +function get_job_id() { + local job_name="${1}" + local query_string="${2:+&${2}}" + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + local max_page=3 + local page=1 + + while true; do + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}" + echoinfo "GET ${url}" + + local job_id + job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last") + [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break + + let "page++" + done + + if [[ "${job_id}" == "" ]]; then + echoerr "The '${job_name}' job ID couldn't be retrieved!" + else + echoinfo "The '${job_name}' job ID is ${job_id}" + echo "${job_id}" + fi +} + +function play_job() { + local job_name="${1}" + local job_id + job_id=$(get_job_id "${job_name}" "scope=manual"); + if [ -z "${job_id}" ]; then return; fi + + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play" + echoinfo "POST ${url}" + + local job_url + job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url") + echoinfo "Manual job '${job_name}' started at: ${job_url}" +} + +function wait_for_job_to_be_done() { + local job_name="${1}" + local query_string="${2}" + local job_id + job_id=$(get_job_id "${job_name}" "${query_string}") + if [ -z "${job_id}" ]; then return; fi + + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + echoinfo "Waiting for the '${job_name}' job to finish..." + + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}" + echoinfo "GET ${url}" + + # In case the job hasn't finished yet. Keep trying until the job times out. + local interval=30 + local elapsed_seconds=0 + while true; do + local job_status + job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".status" | sed -e s/\"//g) + [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break + + printf "." + let "elapsed_seconds+=interval" + sleep ${interval} + done + + local elapsed_minutes=$((elapsed_seconds / 60)) + echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes." + + if [[ "${job_status}" == "failed" ]]; then + echoerr "The '${job_name}' failed." + elif [[ "${job_status}" == "manual" ]]; then + echoinfo "The '${job_name}' is manual." + else + echoinfo "The '${job_name}' passed." + fi } function install_api_client_dependencies_with_apk() { -- cgit v1.2.1 From 9870e86f6c9e8899400fe847f741bda6f458a978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 29 Jul 2019 13:24:33 +0200 Subject: Use the --atomic flag when deploying Review Apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- scripts/review_apps/review-apps.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'scripts') diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 4f617b15a6e..408a0595f9e 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -189,17 +189,11 @@ function deploy() { gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell" gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}" - # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade` - if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then - echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG" - delete - fi - create_application_secret HELM_CMD=$(cat << EOF helm upgrade --install \ - --wait \ + --atomic \ --timeout 900 \ --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ --set global.appConfig.enableUsagePing=false \ -- cgit v1.2.1 From 8f501a74bde199723e1f01121a9a56e01aacaf1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 30 Jul 2019 17:33:00 +0200 Subject: Remove the wait_for_review_app_to_be_accessible step from review-deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- scripts/review_apps/review-apps.sh | 25 ------------------------- 1 file changed, 25 deletions(-) (limited to 'scripts') diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 408a0595f9e..4935c1342a3 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -340,31 +340,6 @@ function display_deployment_debug() { fi } -function wait_for_review_app_to_be_accessible() { - echoinfo "Waiting for the Review App at ${CI_ENVIRONMENT_URL} to be accessible..." true - - local interval=5 - local elapsed_seconds=0 - local max_seconds=$((2 * 60)) - while true; do - local review_app_http_code - review_app_http_code=$(curl --silent --output /dev/null --max-time 5 --write-out "%{http_code}" "${CI_ENVIRONMENT_URL}/users/sign_in") - if [[ "${review_app_http_code}" -eq "200" ]] || [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then - break - fi - - let "elapsed_seconds+=interval" - sleep ${interval} - done - - if [[ "${review_app_http_code}" -eq "200" ]]; then - echoinfo "The Review App at ${CI_ENVIRONMENT_URL} is ready after ${elapsed_seconds} seconds!" - else - echoerr "The Review App at ${CI_ENVIRONMENT_URL} isn't ready after ${max_seconds} seconds of polling..." - exit 1 - fi -} - function add_license() { if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi -- cgit v1.2.1 From 4541b44c3652ee12acfe18fe0d705ce66f362bdc Mon Sep 17 00:00:00 2001 From: Winnie Hellmann Date: Thu, 8 Aug 2019 07:31:43 +0000 Subject: Add script for frontend test file coverage --- scripts/frontend/file_test_coverage.js | 88 ++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 scripts/frontend/file_test_coverage.js (limited to 'scripts') diff --git a/scripts/frontend/file_test_coverage.js b/scripts/frontend/file_test_coverage.js new file mode 100755 index 00000000000..7d1eb45d4bc --- /dev/null +++ b/scripts/frontend/file_test_coverage.js @@ -0,0 +1,88 @@ +#!/usr/bin/env node + +/** + * Counts the number of frontend test files and compares them against the number of application files. + * + * Example output: + * + * Source files: 1551 + * Test files: 716 + * Coverage: 46.16% + */ + +const fs = require('fs'); +const path = require('path'); + +const sourceDirectories = ['app/assets/javascripts']; +const testDirectories = ['spec/javascripts', 'spec/frontend']; + +if (fs.existsSync('ee')) { + sourceDirectories.forEach(dir => { + sourceDirectories.push(`ee/${dir}`); + }); + + testDirectories.forEach(dir => { + testDirectories.push(`ee/${dir}`); + }); +} + +let numSourceFiles = 0; +let numTestFiles = 0; + +const isVerbose = process.argv.some(arg => arg === '-v'); + +const countSourceFiles = path => + forEachFileIn(path, fileName => { + if (fileName.endsWith('.vue') || fileName.endsWith('.js')) { + if (isVerbose) { + console.log(`source file: ${fileName}`); + } + + numSourceFiles += 1; + } + }); + +const countTestFiles = path => + forEachFileIn(path, fileName => { + if (fileName.endsWith('_spec.js')) { + if (isVerbose) { + console.log(`test file: ${fileName}`); + } + + numTestFiles += 1; + } + }); + +function forEachFileIn(dirPath, callback) { + fs.readdir(dirPath, (err, files) => { + if (err) { + console.error(err); + } + + if (!files) { + return; + } + + files.forEach(fileName => { + const absolutePath = path.join(dirPath, fileName); + const stats = fs.statSync(absolutePath); + if (stats.isFile()) { + callback(absolutePath); + } else if (stats.isDirectory()) { + forEachFileIn(absolutePath, callback); + } + }); + }); +} + +console.log(`Source directories: ${sourceDirectories.join(', ')}`); +console.log(`Test directories: ${testDirectories.join(', ')}`); + +sourceDirectories.forEach(countSourceFiles); +testDirectories.forEach(countTestFiles); + +process.on('exit', () => { + console.log(`Source files: ${numSourceFiles}`); + console.log(`Test files: ${numTestFiles}`); + console.log(`Coverage: ${((100 * numTestFiles) / numSourceFiles).toFixed(2)}%`); +}); -- cgit v1.2.1 From a74396dcc5e372c0b6a23fd47db22ebbeb8386d7 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 8 Aug 2019 21:33:20 -0700 Subject: Add Gitaly and Rugged call timing in Sidekiq logs This will help identify Sidekiq jobs that invoke excessive number of filesystem access. The timing data is stored in `RequestStore`, but this is only active within the middleware and is not directly accessible to the Sidekiq logger. However, it is possible for the middleware to modify the job hash to pass this data along to the logger. --- scripts/lint-rugged | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/lint-rugged b/scripts/lint-rugged index d862571c1c5..1b3fb54f70b 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -19,6 +19,7 @@ ALLOWED = [ 'config/initializers/peek.rb', 'config/initializers/lograge.rb', 'lib/gitlab/grape_logging/loggers/perf_logger.rb', + 'lib/gitlab/instrumentation_helper.rb', 'lib/gitlab/rugged_instrumentation.rb', 'lib/peek/views/rugged.rb' ].freeze -- cgit v1.2.1 From bd3699dc6ebc46410839667e6dcfc4038abf9a24 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 9 Aug 2019 15:34:06 +0200 Subject: Removed duplicate functions from utils.sh This change was made in EE in MR https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/14802, but this was not backported to CE. --- scripts/utils.sh | 126 ------------------------------------------------------- 1 file changed, 126 deletions(-) (limited to 'scripts') diff --git a/scripts/utils.sh b/scripts/utils.sh index c1bdef5b847..f0f08e2e1c5 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -158,129 +158,3 @@ function wait_for_job_to_be_done() { echoinfo "The '${job_name}' passed." fi } - -function install_api_client_dependencies_with_apk() { - apk add --update openssl curl jq -} - -function install_api_client_dependencies_with_apt() { - apt update && apt install jq -y -} - -function install_gitlab_gem() { - gem install gitlab --no-document -} - -function echoerr() { - local header="${2}" - - if [ -n "${header}" ]; then - printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2; - else - printf "\033[0;31m%s\n\033[0m" "${1}" >&2; - fi -} - -function echoinfo() { - local header="${2}" - - if [ -n "${header}" ]; then - printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2; - else - printf "\033[0;33m%s\n\033[0m" "${1}" >&2; - fi -} - -function get_job_id() { - local job_name="${1}" - local query_string="${2:+&${2}}" - local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" - if [ -z "${api_token}" ]; then - echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." - return - fi - - local max_page=3 - local page=1 - - while true; do - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}" - echoinfo "GET ${url}" - - local job_id - job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last") - [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break - - let "page++" - done - - if [[ "${job_id}" == "" ]]; then - echoerr "The '${job_name}' job ID couldn't be retrieved!" - else - echoinfo "The '${job_name}' job ID is ${job_id}" - echo "${job_id}" - fi -} - -function play_job() { - local job_name="${1}" - local job_id - job_id=$(get_job_id "${job_name}" "scope=manual"); - if [ -z "${job_id}" ]; then return; fi - - local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" - if [ -z "${api_token}" ]; then - echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." - return - fi - - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play" - echoinfo "POST ${url}" - - local job_url - job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url") - echoinfo "Manual job '${job_name}' started at: ${job_url}" -} - -function wait_for_job_to_be_done() { - local job_name="${1}" - local query_string="${2}" - local job_id - job_id=$(get_job_id "${job_name}" "${query_string}") - if [ -z "${job_id}" ]; then return; fi - - local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" - if [ -z "${api_token}" ]; then - echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." - return - fi - - echoinfo "Waiting for the '${job_name}' job to finish..." - - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}" - echoinfo "GET ${url}" - - # In case the job hasn't finished yet. Keep trying until the job times out. - local interval=30 - local elapsed_seconds=0 - while true; do - local job_status - job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".status" | sed -e s/\"//g) - [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break - - printf "." - let "elapsed_seconds+=interval" - sleep ${interval} - done - - local elapsed_minutes=$((elapsed_seconds / 60)) - echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes." - - if [[ "${job_status}" == "failed" ]]; then - echoerr "The '${job_name}' failed." - elif [[ "${job_status}" == "manual" ]]; then - echoinfo "The '${job_name}' is manual." - else - echoinfo "The '${job_name}' passed." - fi -} -- cgit v1.2.1