Merge branch 'ce-52112-fix-review-apps-cleanup-ce' into 'master'

Improve HelmClient and KubernetesClient

See merge request gitlab-org/gitlab-ce!22375

2 files changed, 96 insertions, 20 deletions

diff --git a/lib/quality/helm_client.rb b/lib/quality/helm_client.rb
index 49d953da681..cf1f03b35b5 100644
--- a/lib/quality/helm_client.rb
+++ b/lib/quality/helm_client.rb
@@ -5,9 +5,13 @@ require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
 
 module Quality
   class HelmClient
+    CommandFailedError = Class.new(StandardError)
+
     attr_reader :namespace
 
-    Release = Struct.new(:name, :revision, :last_update, :status, :chart, :namespace) do
+    RELEASE_JSON_ATTRIBUTES = %w[Name Revision Updated Status Chart AppVersion Namespace].freeze
+
+    Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do
       def revision
         @revision ||= self[:revision].to_i
       end
@@ -17,22 +21,24 @@ module Quality
       end
     end
 
-    def initialize(namespace: ENV['KUBE_NAMESPACE'])
+    # A single page of data and the corresponding page number.
+    Page = Struct.new(:releases, :number)
+
+    def initialize(namespace:)
       @namespace = namespace
     end
 
     def releases(args: [])
-      command = ['list', %(--namespace "#{namespace}"), *args]
-
-      run_command(command)
-        .stdout
-        .lines
-        .select { |line| line.include?(namespace) }
-        .map { |line| Release.new(*line.split(/\t/).map(&:strip)) }
+      each_release(args)
     end
 
     def delete(release_name:)
-      run_command(['delete', '--purge', release_name])
+      run_command([
+        'delete',
+        %(--tiller-namespace "#{namespace}"),
+        '--purge',
+        release_name
+      ])
     end
 
     private
@@ -41,7 +47,67 @@ module Quality
       final_command = ['helm', *command].join(' ')
       puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
 
-      Gitlab::Popen.popen_with_detail([final_command])
+      result = Gitlab::Popen.popen_with_detail([final_command])
+
+      if result.status.success?
+        result.stdout.chomp.freeze
+      else
+        raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+      end
+    end
+
+    def raw_releases(args = [])
+      command = [
+        'list',
+        %(--namespace "#{namespace}"),
+        %(--tiller-namespace "#{namespace}" --output json),
+        *args
+      ]
+      json = JSON.parse(run_command(command))
+
+      releases = json['Releases'].map do |json_release|
+        Release.new(*json_release.values_at(*RELEASE_JSON_ATTRIBUTES))
+      end
+
+      [releases, json['Next']]
+    rescue JSON::ParserError => ex
+      puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output
+      [[], nil]
+    end
+
+    # Fetches data from Helm and yields a Page object for every page
+    # of data, without loading all of them into memory.
+    #
+    # method - The Octokit method to use for getting the data.
+    # args - Arguments to pass to the `helm list` command.
+    def each_releases_page(args, &block)
+      return to_enum(__method__, args) unless block_given?
+
+      page = 1
+      offset = ''
+
+      loop do
+        final_args = args.dup
+        final_args << "--offset #{offset}" unless offset.to_s.empty?
+        collection, offset = raw_releases(final_args)
+
+        yield Page.new(collection, page += 1)
+
+        break if offset.to_s.empty?
+      end
+    end
+
+    # Iterates over all of the releases.
+    #
+    # args - Any arguments to pass to the `helm list` command.
+    def each_release(args, &block)
+      return to_enum(__method__, args) unless block_given?
+
+      each_releases_page(args) do |page|
+        page.releases.each do |release|
+          yield release
+        end
+      end
     end
   end
 end
diff --git a/lib/quality/kubernetes_client.rb b/lib/quality/kubernetes_client.rb
index e366a688e3e..2ff9e811425 100644
--- a/lib/quality/kubernetes_client.rb
+++ b/lib/quality/kubernetes_client.rb
@@ -4,19 +4,22 @@ require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
 
 module Quality
   class KubernetesClient
+    CommandFailedError = Class.new(StandardError)
+
     attr_reader :namespace
 
-    def initialize(namespace: ENV['KUBE_NAMESPACE'])
+    def initialize(namespace:)
       @namespace = namespace
     end
 
     def cleanup(release_name:)
-      command = ['kubectl']
-      command << %(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1)
-      command << '|' << %(grep "#{release_name}")
-      command << '|' << "awk '{print $1}'"
-      command << '|' << %(xargs kubectl -n "#{namespace}" delete)
-      command << '||' << 'true'
+      command = [
+        %(--namespace "#{namespace}"),
+        'delete',
+        'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa',
+        '--now',
+        %(-l release="#{release_name}")
+      ]
 
       run_command(command)
     end
@@ -24,9 +27,16 @@ module Quality
     private
 
     def run_command(command)
-      puts "Running command: `#{command.join(' ')}`" # rubocop:disable Rails/Output
+      final_command = ['kubectl', *command].join(' ')
+      puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+
+      result = Gitlab::Popen.popen_with_detail([final_command])
 
-      Gitlab::Popen.popen_with_detail(command)
+      if result.status.success?
+        result.stdout.chomp.freeze
+      else
+        raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+      end
     end
   end
 end