Add latest changes from gitlab-org/gitlab@master

5 files changed, 108 insertions, 5 deletions

diff --git a/qa/qa/service/praefect_manager.rb b/qa/qa/service/praefect_manager.rb
index c2eb50a4f7f..8ffb7c47652 100644
--- a/qa/qa/service/praefect_manager.rb
+++ b/qa/qa/service/praefect_manager.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'digest'
+
 module QA
   module Service
     class PraefectManager
@@ -68,7 +70,7 @@ module QA
 
       def stop_secondary_node
         stop_node(@secondary_node)
-        wait_until_node_is_removed_from_healthy_storages(@stop_secondary_node)
+        wait_until_node_is_removed_from_healthy_storages(@secondary_node)
       end
 
       def start_secondary_node
@@ -106,6 +108,8 @@ module QA
       end
 
       def stop_node(name)
+        return if node_state(name) == 'paused'
+
         shell "docker pause #{name}"
       end
 
@@ -200,7 +204,7 @@ module QA
         start_node(@primary_node)
         start_node(@secondary_node)
         start_node(@tertiary_node)
-        start_node(@praefect)
+        start_praefect
 
         wait_for_health_check_all_nodes
       end
@@ -281,6 +285,48 @@ module QA
         end
       end
 
+      def praefect_dataloss_information(project_id)
+        dataloss_info = []
+        cmd = "docker exec #{@praefect} praefect -config /var/opt/gitlab/praefect/config.toml dataloss --partially-unavailable=true"
+        shell(cmd) { |line| dataloss_info << line.strip }
+
+        # Expected will have a record for each repository in the storage, in the following format
+        #   @hashed/bc/52/bc52dd634277c4a34a2d6210994a9a5e2ab6d33bb4a3a8963410e00ca6c15a02.git:
+        #     Primary: gitaly1
+        #       In-Sync Storages:
+        #         gitaly1, assigned host
+        #         gitaly3, assigned host
+        #       Outdated Storages:
+        #         gitaly2 is behind by 1 change or less, assigned host
+        #
+        # Alternatively, if all repositories are in sync, a concise message is returned
+        #   Virtual storage: default
+        #     All repositories are fully available on all assigned storages!
+
+        # extract the relevant project under test info if it is identified
+        start_index = dataloss_info.index { |line| line.include?("#{Digest::SHA256.hexdigest(project_id.to_s)}.git") }
+        unless start_index.nil?
+          dataloss_info = dataloss_info[start_index, 7]
+        end
+
+        dataloss_info&.each { |info| QA::Runtime::Logger.debug(info) }
+        dataloss_info
+      end
+
+      def praefect_dataloss_info_for_project(project_id)
+        dataloss_info = []
+        Support::Retrier.retry_until(max_duration: 60) do
+          dataloss_info = praefect_dataloss_information(project_id)
+          dataloss_info.include?("#{Digest::SHA256.hexdigest(project_id.to_s)}.git")
+        end
+      end
+
+      def wait_for_project_synced_across_all_storages(project_id)
+        Support::Retrier.retry_until(max_duration: 60) do
+          praefect_dataloss_information(project_id).include?('All repositories are fully available on all assigned storages!')
+        end
+      end
+
       def wait_for_health_check_all_nodes
         wait_for_gitaly_health_check(@primary_node)
         wait_for_gitaly_health_check(@secondary_node)
diff --git a/qa/qa/specs/features/api/3_create/gitaly/praefect_dataloss_spec.rb b/qa/qa/specs/features/api/3_create/gitaly/praefect_dataloss_spec.rb
new file mode 100644
index 00000000000..6e2a34afb3e
--- /dev/null
+++ b/qa/qa/specs/features/api/3_create/gitaly/praefect_dataloss_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module QA
+  RSpec.describe 'Create' do
+    context 'Praefect dataloss commands', :orchestrated, :gitaly_cluster do
+      let(:praefect_manager) { Service::PraefectManager.new }
+
+      let(:project) do
+        Resource::Project.fabricate! do |project|
+          project.name = 'gitaly_cluster-dataloss-project'
+          project.initialize_with_readme = true
+        end
+      end
+
+      before do
+        praefect_manager.start_all_nodes
+      end
+
+      it 'confirms that changes are synced across all storages', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/352691' do
+        expect { praefect_manager.praefect_dataloss_information(project.id) }
+        .to(eventually_include('All repositories are fully available on all assigned storages!')
+        .within(max_duration: 60))
+      end
+
+      it 'identifies how many changes are not in sync across storages', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/352692' do
+        # Ensure our test repository is replicated and in a consistent state prior to test
+        praefect_manager.wait_for_project_synced_across_all_storages(project.id)
+
+        # testing for gitaly2 'out of sync'
+        praefect_manager.stop_secondary_node
+
+        number_of_changes = 3
+        1.upto(number_of_changes) do |i|
+          Resource::Repository::Commit.fabricate_via_api! do |commit|
+            commit.project = project
+            commit.branch = "newbranch-#{SecureRandom.hex(8)}"
+            commit.start_branch = project.default_branch
+            commit.commit_message = 'Add new file'
+            commit.add_files([
+              { file_path: "new_file-#{SecureRandom.hex(8)}.txt", content: 'new file' }
+            ])
+          end
+        end
+
+        # testing for gitaly3 'in sync' but marked unhealthy
+        praefect_manager.stop_tertiary_node
+
+        project_data_loss = praefect_manager.praefect_dataloss_information(project.id)
+        aggregate_failures "validate dataloss identified" do
+          expect(project_data_loss).to include('gitaly1, assigned host')
+          expect(project_data_loss).to include("gitaly2 is behind by #{number_of_changes} changes or less, assigned host, unhealthy")
+          expect(project_data_loss).to include('gitaly3, assigned host, unhealthy')
+        end
+      end
+    end
+  end
+end
diff --git a/qa/qa/specs/features/browser_ui/14_non_devops/service_ping_default_enabled_spec.rb b/qa/qa/specs/features/browser_ui/14_non_devops/service_ping_default_enabled_spec.rb
index a7c846b2173..bb4e0d71710 100644
--- a/qa/qa/specs/features/browser_ui/14_non_devops/service_ping_default_enabled_spec.rb
+++ b/qa/qa/specs/features/browser_ui/14_non_devops/service_ping_default_enabled_spec.rb
@@ -2,7 +2,7 @@
 
 module QA
   RSpec.describe 'Service ping default enabled' do
-    context 'When using default enabled from gitlab.yml config', :requires_admin do
+    context 'When using default enabled from gitlab.yml config', :requires_admin, except: { job: 'review-qa-*' } do
       before do
         Flow::Login.sign_in_as_admin
 
diff --git a/qa/qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb b/qa/qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb
index e9d8e35478f..96e85139e78 100644
--- a/qa/qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb
+++ b/qa/qa/specs/features/browser_ui/3_create/web_ide/open_fork_in_web_ide_spec.rb
@@ -11,7 +11,7 @@ module QA
       end
 
       context 'when a user does not have permissions to commit to the project' do
-        let(:user) { Resource::User.fabricate_or_use(Runtime::Env.gitlab_qa_username_6, Runtime::Env.gitlab_qa_password_6) }
+        let(:user) { Resource::User.fabricate_or_use(Runtime::Env.gitlab_qa_username_2, Runtime::Env.gitlab_qa_password_2) }
 
         context 'when no fork is present' do
           it 'suggests to create a fork when a user clicks Web IDE in the main project', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347823' do
diff --git a/qa/qa/tools/delete_test_resources.rb b/qa/qa/tools/delete_test_resources.rb
index 31499559481..6f63c56f736 100644
--- a/qa/qa/tools/delete_test_resources.rb
+++ b/qa/qa/tools/delete_test_resources.rb
@@ -64,7 +64,7 @@ module QA
         transformed_values = resources.transform_values! do |v|
           v.reject do |attributes|
             attributes['info'] == "with full_path 'gitlab-qa-sandbox-group'" ||
-              attributes['http_method'] == 'get' && !attributes['info'].include?("with username 'qa-") ||
+              attributes['http_method'] == 'get' && !attributes['info']&.include?("with username 'qa-") ||
               attributes['api_path'] == 'Cannot find resource API path'
           end
         end