From eedf43d69bb2d3e5ed73b751565beb5d1badd8c7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 11 Oct 2017 14:48:28 +0200 Subject: Set pipeline config source attribute in a build step --- spec/support/stub_gitlab_calls.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'spec/support') diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index 5f22d886910..c1618f5086c 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -21,6 +21,12 @@ module StubGitlabCalls allow_any_instance_of(Ci::Pipeline).to receive(:ci_yaml_file) { ci_yaml } end + def stub_repository_ci_yaml_file(sha:, path: '.gitlab-ci.yml') + allow_any_instance_of(Repository) + .to receive(:gitlab_ci_yml_for).with(sha, path) + .and_return(gitlab_ci_yaml) + end + def stub_ci_builds_disabled allow_any_instance_of(Project).to receive(:builds_enabled?).and_return(false) end -- cgit v1.2.1 From 4dfe26cd8b6863b7e6c81f5c280cdafe9b6e17b6 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 13 Oct 2017 18:50:36 +0200 Subject: Rewrite the GitHub importer from scratch Prior to this MR there were two GitHub related importers: * Github::Import: the main importer used for GitHub projects * Gitlab::GithubImport: importer that's somewhat confusingly used for importing Gitea projects (apparently they have a compatible API) This MR renames the Gitea importer to Gitlab::LegacyGithubImport and introduces a new GitHub importer in the Gitlab::GithubImport namespace. This new GitHub importer uses Sidekiq for importing multiple resources in parallel, though it also has the ability to import data sequentially should this be necessary. The new code is spread across the following directories: * lib/gitlab/github_import: this directory contains most of the importer code such as the classes used for importing resources. * app/workers/gitlab/github_import: this directory contains the Sidekiq workers, most of which simply use the code from the directory above. * app/workers/concerns/gitlab/github_import: this directory provides a few modules that are included in every GitHub importer worker. == Stages The import work is divided into separate stages, with each stage importing a specific set of data. Stages will schedule the work that needs to be performed, followed by scheduling a job for the "AdvanceStageWorker" worker. This worker will periodically check if all work is completed and schedule the next stage if this is the case. If work is not yet completed this worker will reschedule itself. Using this approach we don't have to block threads by calling `sleep()`, as doing so for large projects could block the thread from doing any work for many hours. == Retrying Work Workers will reschedule themselves whenever necessary. For example, hitting the GitHub API's rate limit will result in jobs rescheduling themselves. These jobs are not processed until the rate limit has been reset. == User Lookups Part of the importing process involves looking up user details in the GitHub API so we can map them to GitLab users. The old importer used an in-memory cache, but this obviously doesn't work when the work is spread across different threads. The new importer uses a Redis cache and makes sure we only perform API/database calls if absolutely necessary. Frequently used keys are refreshed, and lookup misses are also cached; removing the need for performing API/database calls if we know we don't have the data we're looking for. == Performance & Models The new importer in various places uses raw INSERT statements (as generated by `Gitlab::Database.bulk_insert`) instead of using Rails models. This allows us to bypass any validations and callbacks, drastically reducing the number of SQL queries and Gitaly RPC calls necessary to import projects. To ensure the code produces valid data the corresponding tests check if the produced rows are valid according to the model validation rules. --- .../githubish_import_controller_shared_examples.rb | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'spec/support') diff --git a/spec/support/controllers/githubish_import_controller_shared_examples.rb b/spec/support/controllers/githubish_import_controller_shared_examples.rb index b23d81a226a..a0839eefe6c 100644 --- a/spec/support/controllers/githubish_import_controller_shared_examples.rb +++ b/spec/support/controllers/githubish_import_controller_shared_examples.rb @@ -14,7 +14,7 @@ shared_examples 'a GitHub-ish import controller: POST personal_access_token' do it "updates access token" do token = 'asdfasdf9876' - allow_any_instance_of(Gitlab::GithubImport::Client) + allow_any_instance_of(Gitlab::LegacyGithubImport::Client) .to receive(:user).and_return(true) post :personal_access_token, personal_access_token: token @@ -79,7 +79,7 @@ shared_examples 'a GitHub-ish import controller: GET status' do end it "handles an invalid access token" do - allow_any_instance_of(Gitlab::GithubImport::Client) + allow_any_instance_of(Gitlab::LegacyGithubImport::Client) .to receive(:repos).and_raise(Octokit::Unauthorized) get :status @@ -110,7 +110,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do context "when the repository owner is the provider user" do context "when the provider user and GitLab user's usernames match" do it "takes the current user's namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -122,7 +122,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do let(:provider_username) { "someone_else" } it "takes the current user's namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -149,7 +149,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it "takes the existing namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, existing_namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -161,7 +161,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do it "creates a project using user's namespace" do create(:user, username: other_username) - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -173,14 +173,14 @@ shared_examples 'a GitHub-ish import controller: POST create' do context "when a namespace with the provider user's username doesn't exist" do context "when current user can create namespaces" do it "creates the namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).and_return(double(execute: true)) expect { post :create, target_namespace: provider_repo.name, format: :js }.to change(Namespace, :count).by(1) end it "takes the new namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, an_instance_of(Group), user, access_params, type: provider) .and_return(double(execute: true)) @@ -194,14 +194,14 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it "doesn't create the namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).and_return(double(execute: true)) expect { post :create, format: :js }.not_to change(Namespace, :count) end it "takes the current user's namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -219,7 +219,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'takes the selected namespace and name' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, test_namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -227,7 +227,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'takes the selected name and default namespace' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -245,7 +245,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'takes the selected namespace and name' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, nested_namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -257,7 +257,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do let(:test_name) { 'test_name' } it 'takes the selected namespace and name' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) @@ -265,7 +265,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'creates the namespaces' do - allow(Gitlab::GithubImport::ProjectCreator) + allow(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) @@ -274,7 +274,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'new namespace has the right parent' do - allow(Gitlab::GithubImport::ProjectCreator) + allow(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) @@ -289,7 +289,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do let!(:parent_namespace) { create(:group, name: 'foo', owner: user) } it 'takes the selected namespace and name' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) @@ -297,7 +297,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'creates the namespaces' do - allow(Gitlab::GithubImport::ProjectCreator) + allow(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) -- cgit v1.2.1 From 20ac30a705f4edd22efd934ecf68b58557f868db Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 1 Nov 2017 09:25:49 +0000 Subject: Merge branch '36099-api-responses-missing-x-content-type-options-header' into '10-1-stable' Include X-Content-Type-Options (XCTO) header into API responses See merge request gitlab/gitlabhq!2211 (cherry picked from commit 6c818e77f2abeef2dd7b17a269611b018701fa79) e087e075 Include X-Content-Type-Options (XCTO) header into API responses --- spec/support/matchers/security_header_matcher.rb | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 spec/support/matchers/security_header_matcher.rb (limited to 'spec/support') diff --git a/spec/support/matchers/security_header_matcher.rb b/spec/support/matchers/security_header_matcher.rb new file mode 100644 index 00000000000..f8518d13ebb --- /dev/null +++ b/spec/support/matchers/security_header_matcher.rb @@ -0,0 +1,5 @@ +RSpec::Matchers.define :include_security_headers do |expected| + match do |actual| + expect(actual.headers).to include('X-Content-Type-Options') + end +end -- cgit v1.2.1 From 6545e56d929d7e3df4c176e8f2f0c64c784857a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Tue, 14 Nov 2017 19:53:32 +0100 Subject: we need to disable gitaly for some tests --- spec/support/gitaly.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'spec/support') diff --git a/spec/support/gitaly.rb b/spec/support/gitaly.rb index 89fb362cf14..1512b3e0620 100644 --- a/spec/support/gitaly.rb +++ b/spec/support/gitaly.rb @@ -1,6 +1,10 @@ RSpec.configure do |config| config.before(:each) do |example| - next if example.metadata[:skip_gitaly_mock] - allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(true) + if example.metadata[:disable_gitaly] + allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(false) + else + next if example.metadata[:skip_gitaly_mock] + allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(true) + end end end -- cgit v1.2.1 From 181cd299f9e06223e8338e93b1c318c671ccb1aa Mon Sep 17 00:00:00 2001 From: Jacopo Date: Tue, 14 Nov 2017 10:02:39 +0100 Subject: Adds Rubocop rule for line break after guard clause Adds a rubocop rule (with autocorrect) to ensure line break after guard clauses. --- spec/support/fixture_helpers.rb | 1 + spec/support/generate-seed-repo-rb | 1 + spec/support/gitaly.rb | 1 + 3 files changed, 3 insertions(+) (limited to 'spec/support') diff --git a/spec/support/fixture_helpers.rb b/spec/support/fixture_helpers.rb index 5515c355cea..128aaaf25fe 100644 --- a/spec/support/fixture_helpers.rb +++ b/spec/support/fixture_helpers.rb @@ -1,6 +1,7 @@ module FixtureHelpers def fixture_file(filename) return '' if filename.blank? + File.read(expand_fixture_path(filename)) end diff --git a/spec/support/generate-seed-repo-rb b/spec/support/generate-seed-repo-rb index ef3c8e7087f..4ee33f9725b 100755 --- a/spec/support/generate-seed-repo-rb +++ b/spec/support/generate-seed-repo-rb @@ -33,6 +33,7 @@ end def capture!(cmd, dir) output = IO.popen(cmd, 'r', chdir: dir) { |io| io.read } raise "command failed with #{$?}: #{cmd.join(' ')}" unless $?.success? + output.chomp end diff --git a/spec/support/gitaly.rb b/spec/support/gitaly.rb index 1512b3e0620..c7e8a39a617 100644 --- a/spec/support/gitaly.rb +++ b/spec/support/gitaly.rb @@ -4,6 +4,7 @@ RSpec.configure do |config| allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(false) else next if example.metadata[:skip_gitaly_mock] + allow(Gitlab::GitalyClient).to receive(:feature_enabled?).and_return(true) end end -- cgit v1.2.1 From 59502122bdf928e58a0592da0610f2a94bb5eeb4 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 Nov 2017 10:11:45 -0600 Subject: Fix reply quote keyboard shortcut on MRs Fix https://gitlab.com/gitlab-org/gitlab-ce/issues/40295 --- spec/support/selection_helper.rb | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 spec/support/selection_helper.rb (limited to 'spec/support') diff --git a/spec/support/selection_helper.rb b/spec/support/selection_helper.rb new file mode 100644 index 00000000000..b4725b137b2 --- /dev/null +++ b/spec/support/selection_helper.rb @@ -0,0 +1,6 @@ +module SelectionHelper + def select_element(selector) + find(selector) + execute_script("let range = document.createRange(); let sel = window.getSelection(); range.selectNodeContents(document.querySelector('#{selector}')); sel.addRange(range);") + end +end -- cgit v1.2.1 From 52b6cbcb9df7e2c968da3e9a9779500cacfac465 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 20 Nov 2017 09:18:15 -0800 Subject: Add QUERY_RECORDER_DEBUG environment variable to improve performance debugging --- spec/support/query_recorder.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'spec/support') diff --git a/spec/support/query_recorder.rb b/spec/support/query_recorder.rb index ba0b805caad..cd67b517ea0 100644 --- a/spec/support/query_recorder.rb +++ b/spec/support/query_recorder.rb @@ -8,7 +8,14 @@ module ActiveRecord ActiveSupport::Notifications.subscribed(method(:callback), 'sql.active_record', &block) end + def show_backtrace(values) + Rails.logger.debug("QueryRecorder SQL: #{values[:sql]}") + caller.each { |line| Rails.logger.debug(" --> #{line}") } + end + def callback(name, start, finish, message_id, values) + show_backtrace(values) if ENV['QUERY_RECORDER_DEBUG'] + if values[:name]&.include?("CACHE") @cached << values[:sql] elsif !values[:name]&.include?("SCHEMA") -- cgit v1.2.1 From db1925f917b30571e0e9d4de63417db90dc693d7 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Wed, 22 Nov 2017 11:12:47 +0000 Subject: Improve output for extra queries in specs Previously, this used `Array#-`, which would remove all queries that matches the query text in the original set. However, sometimes we have a problem with parameterised queries, where the query text is identical both times, so we'd run a query N times instead of once, and it would be hidden from the output. Replace the logic to only remove a given query N times from the actual log, where N is the number of times it appears in the expected log. --- spec/support/query_recorder.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'spec/support') diff --git a/spec/support/query_recorder.rb b/spec/support/query_recorder.rb index ba0b805caad..a771b50ec4c 100644 --- a/spec/support/query_recorder.rb +++ b/spec/support/query_recorder.rb @@ -69,10 +69,17 @@ RSpec::Matchers.define :exceed_query_limit do |expected| @recorder.count end + def count_queries(queries) + queries.each_with_object(Hash.new(0)) { |query, counts| counts[query] += 1 } + end + def log_message if expected.is_a?(ActiveRecord::QueryRecorder) - extra_queries = (expected.log - @recorder.log).join("\n\n") - "Extra queries: \n\n #{extra_queries}" + counts = count_queries(expected.log) + extra_queries = @recorder.log.reject { |query| counts[query] -= 1 unless counts[query].zero? } + extra_queries_display = count_queries(extra_queries).map { |query, count| "[#{count}] #{query}" } + + (['Extra queries:'] + extra_queries_display).join("\n\n") else @recorder.log_message end -- cgit v1.2.1 From 4cfcc97544c231c2baf8dc3ab232ed394355b62c Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Thu, 23 Nov 2017 10:48:57 +0000 Subject: Fix encoding bugs in Gitlab::Git::User --- spec/support/matchers/be_a_binary_string.rb | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 spec/support/matchers/be_a_binary_string.rb (limited to 'spec/support') diff --git a/spec/support/matchers/be_a_binary_string.rb b/spec/support/matchers/be_a_binary_string.rb new file mode 100644 index 00000000000..f041ae76167 --- /dev/null +++ b/spec/support/matchers/be_a_binary_string.rb @@ -0,0 +1,9 @@ +RSpec::Matchers.define :be_a_binary_string do |_| + match do |actual| + actual.is_a?(String) && actual.encoding == Encoding.find('ASCII-8BIT') + end + + description do + "be a String with binary encoding" + end +end -- cgit v1.2.1 From 257fd5713485a05460a9170190100643199a7e48 Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Thu, 23 Nov 2017 13:16:14 +0000 Subject: Allow password authentication to be disabled entirely --- spec/support/matchers/have_gitlab_http_status.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'spec/support') diff --git a/spec/support/matchers/have_gitlab_http_status.rb b/spec/support/matchers/have_gitlab_http_status.rb index 3198f1b9edd..e7e418cdde4 100644 --- a/spec/support/matchers/have_gitlab_http_status.rb +++ b/spec/support/matchers/have_gitlab_http_status.rb @@ -8,7 +8,11 @@ RSpec::Matchers.define :have_gitlab_http_status do |expected| end failure_message do |actual| + # actual can be either an ActionDispatch::TestResponse (which uses #response_code) + # or a Capybara::Session (which uses #status_code) + response_code = actual.try(:response_code) || actual.try(:status_code) + "expected the response to have status code #{expected.inspect}" \ - " but it was #{actual.response_code}. The response was: #{actual.body}" + " but it was #{response_code}. The response was: #{actual.body}" end end -- cgit v1.2.1 From 96106287db2b6403809b238689afb592da5e2abf Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Fri, 24 Nov 2017 12:43:02 +0000 Subject: Deduplicate protected ref human_access_levels Previously these were duplicated so they could be different for push/merge, but this was no longer necessary after https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11232 --- spec/support/protected_tags/access_control_ce_shared_examples.rb | 2 +- .../shared_examples/features/protected_branches_access_control_ce.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'spec/support') diff --git a/spec/support/protected_tags/access_control_ce_shared_examples.rb b/spec/support/protected_tags/access_control_ce_shared_examples.rb index 2770cdcbefc..71eec9f3217 100644 --- a/spec/support/protected_tags/access_control_ce_shared_examples.rb +++ b/spec/support/protected_tags/access_control_ce_shared_examples.rb @@ -1,5 +1,5 @@ RSpec.shared_examples "protected tags > access control > CE" do - ProtectedTag::CreateAccessLevel.human_access_levels.each do |(access_type_id, access_type_name)| + ProtectedRefAccess::HUMAN_ACCESS_LEVELS.each do |(access_type_id, access_type_name)| it "allows creating protected tags that #{access_type_name} can create" do visit project_protected_tags_path(project) diff --git a/spec/support/shared_examples/features/protected_branches_access_control_ce.rb b/spec/support/shared_examples/features/protected_branches_access_control_ce.rb index 5fde91512da..17f319f49e9 100644 --- a/spec/support/shared_examples/features/protected_branches_access_control_ce.rb +++ b/spec/support/shared_examples/features/protected_branches_access_control_ce.rb @@ -1,5 +1,5 @@ shared_examples "protected branches > access control > CE" do - ProtectedBranch::PushAccessLevel.human_access_levels.each do |(access_type_id, access_type_name)| + ProtectedRefAccess::HUMAN_ACCESS_LEVELS.each do |(access_type_id, access_type_name)| it "allows creating protected branches that #{access_type_name} can push to" do visit project_protected_branches_path(project) @@ -44,7 +44,7 @@ shared_examples "protected branches > access control > CE" do end end - ProtectedBranch::MergeAccessLevel.human_access_levels.each do |(access_type_id, access_type_name)| + ProtectedRefAccess::HUMAN_ACCESS_LEVELS.each do |(access_type_id, access_type_name)| it "allows creating protected branches that #{access_type_name} can merge to" do visit project_protected_branches_path(project) -- cgit v1.2.1 From 45f2d0af4173ae8e06b548c4bef1fabe14353c85 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 22 Nov 2017 18:31:07 +0900 Subject: Add test suit for platform::kubernetes --- .../additional_metrics_shared_examples.rb | 28 ++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'spec/support') diff --git a/spec/support/prometheus/additional_metrics_shared_examples.rb b/spec/support/prometheus/additional_metrics_shared_examples.rb index 620fa37d455..7e20b4e0232 100644 --- a/spec/support/prometheus/additional_metrics_shared_examples.rb +++ b/spec/support/prometheus/additional_metrics_shared_examples.rb @@ -41,16 +41,30 @@ RSpec.shared_examples 'additional metrics query' do end describe 'project has Kubernetes service' do - let(:project) { create(:kubernetes_project) } - let(:environment) { create(:environment, slug: 'environment-slug', project: project) } - let(:kube_namespace) { project.kubernetes_service.actual_namespace } + shared_examples 'correct behavior with metrics' do + let(:environment) { create(:environment, slug: 'environment-slug', project: project) } + let(:kube_namespace) { project.kubernetes_service.actual_namespace } - it_behaves_like 'query context containing environment slug and filter' + it_behaves_like 'query context containing environment slug and filter' - it 'query context contains kube_namespace' do - expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: kube_namespace)) + it 'query context contains kube_namespace' do + expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: kube_namespace)) - subject.query(*query_params) + subject.query(*query_params) + end + end + + context 'when user configured kubernetes from Integration > Kubernetes' do + let(:project) { create(:kubernetes_project) } + + it_behaves_like 'correct behavior with metrics' + end + + context 'when user configured kubernetes from CI/CD > Clusters' do + let!(:cluster) { create(:cluster, :project, :provided_by_gcp) } + let(:project) { cluster.project } + + it_behaves_like 'correct behavior with metrics' end end -- cgit v1.2.1 From 53da3d976f3705a87edc50dca41748b5e479fc83 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 27 Nov 2017 22:35:16 +0900 Subject: Replce kubernetes_service and deployment_service to deployment_platform --- spec/support/prometheus/additional_metrics_shared_examples.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'spec/support') diff --git a/spec/support/prometheus/additional_metrics_shared_examples.rb b/spec/support/prometheus/additional_metrics_shared_examples.rb index 7e20b4e0232..f3338259a77 100644 --- a/spec/support/prometheus/additional_metrics_shared_examples.rb +++ b/spec/support/prometheus/additional_metrics_shared_examples.rb @@ -43,7 +43,7 @@ RSpec.shared_examples 'additional metrics query' do describe 'project has Kubernetes service' do shared_examples 'correct behavior with metrics' do let(:environment) { create(:environment, slug: 'environment-slug', project: project) } - let(:kube_namespace) { project.kubernetes_service.actual_namespace } + let(:kube_namespace) { project.deployment_platform.actual_namespace } it_behaves_like 'query context containing environment slug and filter' -- cgit v1.2.1 From c36d7842da24e6726705199f178c1324c634bdaf Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 27 Nov 2017 23:19:16 +0900 Subject: Aling shared_exmaples to "same behavior between KubernetesService and Platform::Kubernetes" --- spec/support/prometheus/additional_metrics_shared_examples.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'spec/support') diff --git a/spec/support/prometheus/additional_metrics_shared_examples.rb b/spec/support/prometheus/additional_metrics_shared_examples.rb index f3338259a77..dbbd4ad4d40 100644 --- a/spec/support/prometheus/additional_metrics_shared_examples.rb +++ b/spec/support/prometheus/additional_metrics_shared_examples.rb @@ -41,7 +41,7 @@ RSpec.shared_examples 'additional metrics query' do end describe 'project has Kubernetes service' do - shared_examples 'correct behavior with metrics' do + shared_examples 'same behavior between KubernetesService and Platform::Kubernetes' do let(:environment) { create(:environment, slug: 'environment-slug', project: project) } let(:kube_namespace) { project.deployment_platform.actual_namespace } @@ -57,14 +57,14 @@ RSpec.shared_examples 'additional metrics query' do context 'when user configured kubernetes from Integration > Kubernetes' do let(:project) { create(:kubernetes_project) } - it_behaves_like 'correct behavior with metrics' + it_behaves_like 'same behavior between KubernetesService and Platform::Kubernetes' end context 'when user configured kubernetes from CI/CD > Clusters' do let!(:cluster) { create(:cluster, :project, :provided_by_gcp) } let(:project) { cluster.project } - it_behaves_like 'correct behavior with metrics' + it_behaves_like 'same behavior between KubernetesService and Platform::Kubernetes' end end -- cgit v1.2.1 From b755c883bcc47382e1115d286062edb25392e25d Mon Sep 17 00:00:00 2001 From: Mike Greiling Date: Tue, 28 Nov 2017 23:47:38 -0600 Subject: add logging back to capybara config and clean up selenium option instantiation --- spec/support/capybara.rb | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'spec/support') diff --git a/spec/support/capybara.rb b/spec/support/capybara.rb index 9f672bc92fc..935b170a0f6 100644 --- a/spec/support/capybara.rb +++ b/spec/support/capybara.rb @@ -7,21 +7,41 @@ require 'selenium-webdriver' # Give CI some extra time timeout = (ENV['CI'] || ENV['CI_SERVER']) ? 60 : 30 -Capybara.javascript_driver = :chrome Capybara.register_driver :chrome do |app| - extra_args = [] - extra_args << 'headless' unless ENV['CHROME_HEADLESS'] =~ /^(false|no|0)$/i - capabilities = Selenium::WebDriver::Remote::Capabilities.chrome( - chromeOptions: { - 'args' => %w[no-sandbox disable-gpu --window-size=1240,1400] + extra_args + # This enables access to logs with `page.driver.manage.get_log(:browser)` + loggingPrefs: { + browser: "ALL", + client: "ALL", + driver: "ALL", + server: "ALL" } ) - Capybara::Selenium::Driver - .new(app, browser: :chrome, desired_capabilities: capabilities) + options = Selenium::WebDriver::Chrome::Options.new + options.add_argument("window-size=1240,1400") + + # Chrome won't work properly in a Docker container in sandbox mode + options.add_argument("no-sandbox") + + # Run headless by default unless CHROME_HEADLESS specified + unless ENV['CHROME_HEADLESS'] =~ /^(false|no|0)$/i + options.add_argument("headless") + + # Chrome documentation says this flag is needed for now + # https://developers.google.com/web/updates/2017/04/headless-chrome#cli + options.add_argument("disable-gpu") + end + + Capybara::Selenium::Driver.new( + app, + browser: :chrome, + desired_capabilities: capabilities, + options: options + ) end +Capybara.javascript_driver = :chrome Capybara.default_max_wait_time = timeout Capybara.ignore_hidden_elements = true -- cgit v1.2.1 From 2ab3031bd35213802e508fef6eebceaaf40cee9b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 15:05:08 -0800 Subject: Refactor, no change in behavior --- spec/support/track_untracked_uploads_helpers.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 spec/support/track_untracked_uploads_helpers.rb (limited to 'spec/support') diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb new file mode 100644 index 00000000000..749c5775bb0 --- /dev/null +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -0,0 +1,12 @@ +module TrackUntrackedUploadsHelpers + def rails_sample_jpg_attrs + @rails_sample_jpg_attrs ||= { + "size" => File.size(rails_sample_file_path), + "checksum" => Digest::SHA256.file(rails_sample_file_path).hexdigest + } + end + + def rails_sample_file_path + Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + end +end -- cgit v1.2.1 From d530085685105e2d7cd6d87ba866756683f0488d Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 23:15:30 -0800 Subject: Refactor specs --- spec/support/track_untracked_uploads_helpers.rb | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'spec/support') diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 749c5775bb0..5b832929602 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -1,12 +1,6 @@ module TrackUntrackedUploadsHelpers - def rails_sample_jpg_attrs - @rails_sample_jpg_attrs ||= { - "size" => File.size(rails_sample_file_path), - "checksum" => Digest::SHA256.file(rails_sample_file_path).hexdigest - } - end - - def rails_sample_file_path - Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + def uploaded_file + fixture_path = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + fixture_file_upload(fixture_path) end end -- cgit v1.2.1 From dd8680a7ae4be279ae1d90f0889317a1e6ee0d95 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 02:36:25 -0800 Subject: Drop temporary tracking table when finished --- spec/support/track_untracked_uploads_helpers.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'spec/support') diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 5b832929602..bb700bc53f1 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -3,4 +3,18 @@ module TrackUntrackedUploadsHelpers fixture_path = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') fixture_file_upload(fixture_path) end + + def recreate_temp_table_if_dropped + TrackUntrackedUploads.new.ensure_temporary_tracking_table_exists + end + + RSpec.configure do |config| + config.after(:each, :temp_table_may_drop) do + recreate_temp_table_if_dropped + end + + config.after(:context, :temp_table_may_drop) do + recreate_temp_table_if_dropped + end + end end -- cgit v1.2.1 From 87529ce5823036d4b9dd9ca412643befc8e490c3 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 05:19:07 -0800 Subject: Move temp table creation into the prepare job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Hopefully fixes spec failures in which the table doesn’t exist * Decouples the background migration from the post-deploy migration, e.g. we could easily run it again even though the table is dropped when finished. --- spec/support/track_untracked_uploads_helpers.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'spec/support') diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index bb700bc53f1..4d4745fd7f4 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -4,17 +4,11 @@ module TrackUntrackedUploadsHelpers fixture_file_upload(fixture_path) end - def recreate_temp_table_if_dropped - TrackUntrackedUploads.new.ensure_temporary_tracking_table_exists + def ensure_temporary_tracking_table_exists + Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists) end - RSpec.configure do |config| - config.after(:each, :temp_table_may_drop) do - recreate_temp_table_if_dropped - end - - config.after(:context, :temp_table_may_drop) do - recreate_temp_table_if_dropped - end + def drop_temp_table_if_exists + ActiveRecord::Base.connection.drop_table(:untracked_files_for_uploads) if ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) end end -- cgit v1.2.1 From ba5697fd809563cb2fd619d7c50362303ab86434 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 8 Nov 2017 10:46:47 +0100 Subject: Fix legacy migration test --- spec/support/test_env.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'spec/support') diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index fff120fcb88..b300b493f86 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -120,6 +120,7 @@ module TestEnv FileUtils.mkdir_p(repos_path) FileUtils.mkdir_p(backup_path) FileUtils.mkdir_p(pages_path) + FileUtils.mkdir_p(artifacts_path) end def clean_gitlab_test_path @@ -233,6 +234,10 @@ module TestEnv Gitlab.config.pages.path end + def artifacts_path + Gitlab.config.artifacts.path + end + # When no cached assets exist, manually hit the root path to create them # # Otherwise they'd be created by the first test, often timing out and -- cgit v1.2.1 From 3d4ba90c5007cca3e8619afe8f40675962a9bc73 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 4 Dec 2017 16:58:44 +0100 Subject: Count occurrences of a specific query in the query recorder. --- spec/support/query_recorder.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'spec/support') diff --git a/spec/support/query_recorder.rb b/spec/support/query_recorder.rb index 369775db462..8cf8f45a8b2 100644 --- a/spec/support/query_recorder.rb +++ b/spec/support/query_recorder.rb @@ -41,7 +41,8 @@ RSpec::Matchers.define :exceed_query_limit do |expected| supports_block_expectations match do |block| - query_count(&block) > expected_count + threshold + @subject_block = block + actual_count > expected_count + threshold end failure_message_when_negated do |actual| @@ -55,6 +56,11 @@ RSpec::Matchers.define :exceed_query_limit do |expected| self end + def for_query(query) + @query = query + self + end + def threshold @threshold.to_i end @@ -68,12 +74,15 @@ RSpec::Matchers.define :exceed_query_limit do |expected| end def actual_count - @recorder.count + @actual_count ||= if @query + recorder.log.select { |recorded| recorded =~ @query }.size + else + recorder.count + end end - def query_count(&block) - @recorder = ActiveRecord::QueryRecorder.new(&block) - @recorder.count + def recorder + @recorder ||= ActiveRecord::QueryRecorder.new(&@subject_block) end def count_queries(queries) -- cgit v1.2.1 From 856447ccd305832c4e9421ab9e81d63eea348203 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Fri, 1 Dec 2017 14:52:16 +0100 Subject: Throttle the number of UPDATEs triggered by touch This throttles the number of UPDATE queries that can be triggered by calling "touch" on a Note, Issue, or MergeRequest. For Note objects we also take care of updating the associated "noteable" relation in a smarter way than Rails does by default. --- spec/support/shared_examples/throttled_touch.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 spec/support/shared_examples/throttled_touch.rb (limited to 'spec/support') diff --git a/spec/support/shared_examples/throttled_touch.rb b/spec/support/shared_examples/throttled_touch.rb new file mode 100644 index 00000000000..4a25bb9b750 --- /dev/null +++ b/spec/support/shared_examples/throttled_touch.rb @@ -0,0 +1,20 @@ +shared_examples_for 'throttled touch' do + describe '#touch' do + it 'updates the updated_at timestamp' do + Timecop.freeze do + subject.touch + expect(subject.updated_at).to eq(Time.zone.now) + end + end + + it 'updates the object at most once per minute' do + first_updated_at = Time.zone.now - (ThrottledTouch::TOUCH_INTERVAL * 2) + second_updated_at = Time.zone.now - (ThrottledTouch::TOUCH_INTERVAL * 1.5) + + Timecop.freeze(first_updated_at) { subject.touch } + Timecop.freeze(second_updated_at) { subject.touch } + + expect(subject.updated_at).to eq(first_updated_at) + end + end +end -- cgit v1.2.1 From 03cba8c0f18f18a14453b17c9f4fa300547f0ab5 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Dec 2017 23:08:45 -0800 Subject: Fix specs after rebase Later migrations added fields to the EE DB which were used by factories which were used in these specs. And in CE on MySQL, a single appearance row is enforced. The migration and migration specs should not depend on the codebase staying the same. --- spec/support/track_untracked_uploads_helpers.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'spec/support') diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 4d4745fd7f4..d05eda08201 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -11,4 +11,10 @@ module TrackUntrackedUploadsHelpers def drop_temp_table_if_exists ActiveRecord::Base.connection.drop_table(:untracked_files_for_uploads) if ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) end + + def create_or_update_appearance(attrs) + a = Appearance.first_or_initialize(title: 'foo', description: 'bar') + a.update!(attrs) + a + end end -- cgit v1.2.1 From c21b488e8357001fa1c250d64b18c74a4c3c41bc Mon Sep 17 00:00:00 2001 From: Takuya Noguchi Date: Mon, 27 Nov 2017 01:08:08 +0900 Subject: Rename GKE as Kubernetes Engine --- spec/support/google_api/cloud_platform_helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'spec/support') diff --git a/spec/support/google_api/cloud_platform_helpers.rb b/spec/support/google_api/cloud_platform_helpers.rb index dabf0db7666..8a073e58db8 100644 --- a/spec/support/google_api/cloud_platform_helpers.rb +++ b/spec/support/google_api/cloud_platform_helpers.rb @@ -63,7 +63,7 @@ module GoogleApi ## # gcloud container clusters create - # https://cloud.google.com/container-engine/reference/rest/v1/projects.zones.clusters/create + # https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.zones.clusters/create # rubocop:disable Metrics/CyclomaticComplexity # rubocop:disable Metrics/PerceivedComplexity def cloud_platform_cluster_body(**options) -- cgit v1.2.1 From f7c18ca31469b199c1a898cef583c9aae99f1375 Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Wed, 6 Dec 2017 12:36:11 +0100 Subject: Support uploads for groups --- .../controllers/uploads_actions_shared_examples.rb | 240 +++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb (limited to 'spec/support') diff --git a/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb b/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb new file mode 100644 index 00000000000..935c08221e0 --- /dev/null +++ b/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb @@ -0,0 +1,240 @@ +shared_examples 'handle uploads' do + let(:user) { create(:user) } + let(:jpg) { fixture_file_upload(Rails.root + 'spec/fixtures/rails_sample.jpg', 'image/jpg') } + let(:txt) { fixture_file_upload(Rails.root + 'spec/fixtures/doc_sample.txt', 'text/plain') } + + describe "POST #create" do + context 'when a user is not authorized to upload a file' do + it 'returns 404 status' do + post :create, params.merge(file: jpg, format: :json) + + expect(response.status).to eq(404) + end + end + + context 'when a user can upload a file' do + before do + sign_in(user) + model.add_developer(user) + end + + context "without params['file']" do + it "returns an error" do + post :create, params.merge(format: :json) + + expect(response).to have_gitlab_http_status(422) + end + end + + context 'with valid image' do + before do + post :create, params.merge(file: jpg, format: :json) + end + + it 'returns a content with original filename, new link, and correct type.' do + expect(response.body).to match '\"alt\":\"rails_sample\"' + expect(response.body).to match "\"url\":\"/uploads" + end + + # NOTE: This is as close as we're getting to an Integration test for this + # behavior. We're avoiding a proper Feature test because those should be + # testing things entirely user-facing, which the Upload model is very much + # not. + it 'creates a corresponding Upload record' do + upload = Upload.last + + aggregate_failures do + expect(upload).to exist + expect(upload.model).to eq(model) + end + end + end + + context 'with valid non-image file' do + before do + post :create, params.merge(file: txt, format: :json) + end + + it 'returns a content with original filename, new link, and correct type.' do + expect(response.body).to match '\"alt\":\"doc_sample.txt\"' + expect(response.body).to match "\"url\":\"/uploads" + end + end + end + end + + describe "GET #show" do + let(:show_upload) do + get :show, params.merge(secret: "123456", filename: "image.jpg") + end + + context "when the model is public" do + before do + model.update_attribute(:visibility_level, Gitlab::VisibilityLevel::PUBLIC) + end + + context "when not signed in" do + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file doesn't exist" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + + context "when signed in" do + before do + sign_in(user) + end + + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file doesn't exist" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + end + + context "when the model is private" do + before do + model.update_attribute(:visibility_level, Gitlab::VisibilityLevel::PRIVATE) + end + + context "when not signed in" do + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + context "when the file is an image" do + before do + allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file is not an image" do + it "redirects to the sign in page" do + show_upload + + expect(response).to redirect_to(new_user_session_path) + end + end + end + + context "when the file doesn't exist" do + it "redirects to the sign in page" do + show_upload + + expect(response).to redirect_to(new_user_session_path) + end + end + end + + context "when signed in" do + before do + sign_in(user) + end + + context "when the user has access to the project" do + before do + model.add_developer(user) + end + + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file doesn't exist" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + + context "when the user doesn't have access to the model" do + context "when the file exists" do + before do + allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg) + allow(jpg).to receive(:exists?).and_return(true) + end + + context "when the file is an image" do + before do + allow_any_instance_of(FileUploader).to receive(:image?).and_return(true) + end + + it "responds with status 200" do + show_upload + + expect(response).to have_gitlab_http_status(200) + end + end + + context "when the file is not an image" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + + context "when the file doesn't exist" do + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + end + end + end + end +end -- cgit v1.2.1 From f1ae1e39ce6b7578c5697c977bc3b52b119301ab Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 13 Nov 2017 16:52:07 +0100 Subject: Move the circuitbreaker check out in a separate process Moving the check out of the general requests, makes sure we don't have any slowdown in the regular requests. To keep the process performing this checks small, the check is still performed inside a unicorn. But that is called from a process running on the same server. Because the checks are now done outside normal request, we can have a simpler failure strategy: The check is now performed in the background every `circuitbreaker_check_interval`. Failures are logged in redis. The failures are reset when the check succeeds. Per check we will try `circuitbreaker_access_retries` times within `circuitbreaker_storage_timeout` seconds. When the number of failures exceeds `circuitbreaker_failure_count_threshold`, we will block access to the storage. After `failure_reset_time` of no checks, we will clear the stored failures. This could happen when the process that performs the checks is not running. --- spec/support/stored_repositories.rb | 21 ++++++++++++++++++++- spec/support/stub_configuration.rb | 2 ++ 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'spec/support') diff --git a/spec/support/stored_repositories.rb b/spec/support/stored_repositories.rb index f3deae0f455..f9121cce985 100644 --- a/spec/support/stored_repositories.rb +++ b/spec/support/stored_repositories.rb @@ -12,6 +12,25 @@ RSpec.configure do |config| raise GRPC::Unavailable.new('Gitaly broken in this spec') end - Gitlab::Git::Storage::CircuitBreaker.reset_all! + # Track the maximum number of failures + first_failure = Time.parse("2017-11-14 17:52:30") + last_failure = Time.parse("2017-11-14 18:54:37") + failure_count = Gitlab::CurrentSettings + .current_application_settings + .circuitbreaker_failure_count_threshold + 1 + cache_key = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}broken:#{Gitlab::Environment.hostname}" + + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, cache_key) + redis.hset(cache_key, :first_failure, first_failure.to_i) + redis.hset(cache_key, :last_failure, last_failure.to_i) + redis.hset(cache_key, :failure_count, failure_count.to_i) + end + end + end + + config.after(:each, :broken_storage) do + Gitlab::Git::Storage.redis.with(&:flushall) end end diff --git a/spec/support/stub_configuration.rb b/spec/support/stub_configuration.rb index 4ead78529c3..b36cf3c544c 100644 --- a/spec/support/stub_configuration.rb +++ b/spec/support/stub_configuration.rb @@ -43,6 +43,8 @@ module StubConfiguration end def stub_storage_settings(messages) + messages.deep_stringify_keys! + # Default storage is always required messages['default'] ||= Gitlab.config.repositories.storages.default messages.each do |storage_name, storage_settings| -- cgit v1.2.1