Add initial support for Kaniko-based build of Docker imagesnative-support-for-kaniko-in-gitlab-ci-yml

7 files changed, 331 insertions, 7 deletions

diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index f290ff3a565..4050b188061 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -9,7 +9,7 @@ module Gitlab
           include Configurable
           include Attributable
 
-          ALLOWED_KEYS = %i[tags script only except type image services
+          ALLOWED_KEYS = %i[tags script kaniko only except type image services
                             allow_failure type stage when start_in artifacts cache
                             dependencies before_script after_script variables
                             environment coverage retry extends].freeze
@@ -17,11 +17,12 @@ module Gitlab
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
             validates :config, presence: true
-            validates :script, presence: true
             validates :name, presence: true
             validates :name, type: Symbol
 
             with_options allow_nil: true do
+              validates :kaniko, type: Hash
+
               validates :tags, array_of_strings: true
               validates :allow_failure, boolean: true
               validates :retry, numericality: { only_integer: true,
@@ -38,6 +39,14 @@ module Gitlab
 
             validates :start_in, duration: { limit: '1 day' }, if: :delayed?
             validates :start_in, absence: true, unless: :delayed?
+
+            validate do
+              errors.add(:config, 'either script or kaniko entry needs to be specified') unless script_or_kaniko_specified?
+            end
+
+            def script_or_kaniko_specified?
+              config.key?(:script) || config.key?(:kaniko)
+            end
           end
 
           entry :before_script, Entry::Script,
@@ -46,6 +55,9 @@ module Gitlab
           entry :script, Entry::Commands,
             description: 'Commands that will be executed in this job.'
 
+          entry :kaniko, Entry::Kaniko,
+            description: 'Configuration for Kaniko based Docker image build'
+
           entry :stage, Entry::Stage,
             description: 'Pipeline stage this job will be executed into.'
 
@@ -82,11 +94,11 @@ module Gitlab
           entry :coverage, Entry::Coverage,
                description: 'Coverage configuration for this job.'
 
-          helpers :before_script, :script, :stage, :type, :after_script,
+          helpers :before_script, :script, :kaniko, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
                   :artifacts, :commands, :environment, :coverage, :retry
 
-          attributes :script, :tags, :allow_failure, :when, :dependencies,
+          attributes :script, :kaniko, :tags, :allow_failure, :when, :dependencies,
                      :retry, :extends, :start_in
 
           def compose!(deps = nil)
@@ -110,7 +122,7 @@ module Gitlab
           end
 
           def commands
-            (before_script_value.to_a + script_value.to_a).join("\n")
+            (before_script_value.to_a + ensure_script_value.to_a).join("\n")
           end
 
           def manual_action?
@@ -143,9 +155,9 @@ module Gitlab
           def to_hash
             { name: name,
               before_script: before_script_value,
-              script: script_value,
+              script: ensure_script_value,
               commands: commands,
-              image: image_value,
+              image: ensure_image_value,
               services: services_value,
               stage: stage_value,
               cache: cache_value,
@@ -160,6 +172,16 @@ module Gitlab
               after_script: after_script_value,
               ignore: ignored? }
           end
+
+          def ensure_script_value
+            return [[script_value], kaniko_value[:script]].flatten if @config.key?(:kaniko)
+            script_value
+          end
+
+          def ensure_image_value
+            return kaniko_value[:image] if @config.key?(:kaniko)
+            image_value
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/entry/kaniko.rb b/lib/gitlab/ci/config/entry/kaniko.rb
new file mode 100644
index 00000000000..3a838540759
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/kaniko.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a configuration of Kaniko-based Docker build.
+        #
+        class Kaniko < Node
+          include Configurable
+          include Validatable
+          include Attributable
+
+          KANIKO_IMAGE = 'gcr.io/kaniko-project/executor:debug'.freeze
+          ALLOWED_KEYS = %i[credentials images].freeze
+
+          attributes ALLOWED_KEYS
+
+          entry :credentials, Entry::KanikoCredentials, description: 'Credentials for Kaniko based Docker image build'
+          entry :images, Entry::KanikoImages, description: 'Definition of images to build'
+
+          helpers :credentials, :images
+
+          validations do
+            validates :config, type: Hash
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            with_options allow_nil: true do
+              validates :credentials, type: Array
+              validates :images, type: Array
+            end
+          end
+
+          def self.default
+            { credentials: Entry::KanikoCredentials.default, images: Entry::KanikoImages.default }
+          end
+
+          def value
+            { script: [[credentials_value], images_value].flatten,
+              image: { name: KANIKO_IMAGE, entrypoint: [''] } }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/kaniko_credential.rb b/lib/gitlab/ci/config/entry/kaniko_credential.rb
new file mode 100644
index 00000000000..aed3f7dc9f7
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/kaniko_credential.rb
@@ -0,0 +1,37 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a Docker Registry credential for Kaniko.
+        #
+        class KanikoCredential < Node
+          include Validatable
+          include Attributable
+
+          DEFAULT_REGISTRY = '$CI_REGISTRY'.freeze
+          DEFAULT_USERNAME = '$CI_REGISTRY_USER'.freeze
+          DEFAULT_PASSWORD = '$CI_REGISTRY_PASSWORD'.freeze
+          ALLOWED_KEYS = %i[registry username password].freeze
+
+          attributes ALLOWED_KEYS
+
+          validations do
+            validates :config, type: Hash
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            validates :registry, type: String, presence: true
+            validates :username, type: String, presence: true
+            validates :password, type: String, presence: true
+          end
+
+          def self.default
+            { registry: DEFAULT_REGISTRY,
+              username: DEFAULT_USERNAME,
+              password: DEFAULT_PASSWORD }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/kaniko_credentials.rb b/lib/gitlab/ci/config/entry/kaniko_credentials.rb
new file mode 100644
index 00000000000..1dc62f319b3
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/kaniko_credentials.rb
@@ -0,0 +1,57 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents Docker Registry credentials for Kaniko.
+        #
+        class KanikoCredentials < Node
+          include Validatable
+
+          validations do
+            validates :config, type: Array
+          end
+
+          def self.default
+            [ Entry::KanikoCredential.default ]
+          end
+
+          def compose!(deps = nil)
+            super do
+              @entries = []
+              @config.each do |config|
+                @entries << Entry::Factory.new(Entry::KanikoCredential)
+                                .value(config)
+                                .create!
+              end
+
+              # Ensure, that default credentials for internal registry are
+              # always added to the configuration
+              @entries << Entry::Factory.new(Entry::KanikoCredential)
+                              .value(Entry::KanikoCredential.default)
+                              .create!
+
+              @entries.each do |entry|
+                entry.compose!(deps)
+              end
+            end
+          end
+
+          def value
+            config = { auth: {} }
+            @entries.each do |credential|
+              config[:auth][credential.registry] = { username: credential.username,
+                                                     password: credential.password }
+            end
+
+            %Q(echo "#{config.to_json.gsub('"', '\"')}" > /kaniko/.docker/config.json)
+          end
+
+          def descendants
+            @entries
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/kaniko_image.rb b/lib/gitlab/ci/config/entry/kaniko_image.rb
new file mode 100644
index 00000000000..aed38969c61
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/kaniko_image.rb
@@ -0,0 +1,89 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a configuration of Docker image build for Kaniko.
+        #
+        class KanikoImage < Node
+          include Validatable
+          include Attributable
+
+          DEFAULT_CONTEXT = '.'.freeze
+          DEFAULT_DOCKERFILE = 'Dockerfile'.freeze
+          DEFAULT_TAGS = ['$CI_REGISTRY_IMAGE'].freeze
+          ALLOWED_KEYS = %i[args context dockerfile tags].freeze
+
+          attributes :args
+
+          def context
+            @config[:context] || DEFAULT_CONTEXT
+          end
+
+          def dockerfile
+            @config[:dockerfile] || DEFAULT_DOCKERFILE
+          end
+
+          def tags
+            @config[:tags] || DEFAULT_TAGS
+          end
+
+          validations do
+            validates :config, type: Hash
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            with_options allow_nil: true do
+              validates :args, type: Hash
+              validates :context, type: String
+              validates :dockerfile, type: String
+              validates :tags, array_of_strings: true
+            end
+          end
+
+          def self.default
+            { context: DEFAULT_CONTEXT,
+              dockerfile: DEFAULT_DOCKERFILE,
+              tags: DEFAULT_TAGS }
+          end
+
+          def value
+            append_context
+            append_dockerfile
+            append_tags
+            append_args
+
+            command.join(' ')
+          end
+
+          private
+
+          def command
+            @command ||= ["/kaniko/executor"]
+          end
+
+          def append_context
+            command << "--context $CI_PROJECT_DIR/#{context}"
+          end
+
+          def append_dockerfile
+            command << "--dockerfile $CI_PROJECT_DIR/#{dockerfile}"
+          end
+
+          def append_args
+            return unless args
+
+            args.each do |arg, value|
+              command << "--build-arg #{arg}=#{value}"
+            end
+          end
+
+          def append_tags
+            tags.each do |tag|
+              command << "--destination #{tag}"
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/kaniko_images.rb b/lib/gitlab/ci/config/entry/kaniko_images.rb
new file mode 100644
index 00000000000..8dc75a266ab
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/kaniko_images.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a configuration of Docker image builds for Kaniko.
+        #
+        class KanikoImages < Node
+          include Validatable
+
+          validations do
+            validates :config, type: Array
+          end
+
+          def self.default
+            [ Entry::KanikoImage.default ]
+          end
+
+          def compose!(deps = nil)
+            super do
+              @entries = []
+              @config.each do |config|
+                @entries << Entry::Factory.new(Entry::KanikoImage)
+                                .value(config)
+                                .create!
+              end
+
+              @entries.each do |entry|
+                entry.compose!(deps)
+              end
+            end
+          end
+
+          def value
+            @entries.map(&:value)
+          end
+
+          def descendants
+            @entries
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb
index 85b23edce9f..df9b471f63e 100644
--- a/spec/lib/gitlab/ci/yaml_processor_spec.rb
+++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb
@@ -330,6 +330,35 @@ module Gitlab
         end
       end
 
+      describe "Kaniko build handling" do
+        let(:config_processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)) }
+
+        let(:config) do
+            {
+              test: { kaniko: { credentials: [ { registry: 'registry.example.com',
+                                                 username: 'username',
+                                                 password: 'password' },
+                                               { registry: 'registry2.example.com',
+                                                 username: 'username2',
+                                                 password: 'password2' }],
+                                images: [ { #context: 'dockerfiles/test-1',
+                                           dockerfile: 'dockerfiles/test-1/Dockerfile',
+                                           tags: %w(registry.example.com/my/image-1 registry.example.com/my/image-2),
+                                           args: { 'ALPINE_VERSION' => '3.8' } },
+                                         { context: 'dockerfiles/test-2',
+                                           dockerfile: 'dockerfiles/test-2/Dockerfile',
+                                           tags: %w(registry.example.com/my/image-3) } ] } }
+            }
+        end
+
+        subject { config_processor.build_attributes('test') }
+
+        it 'prepares custom script' do
+          pp subject[:options]
+          puts subject[:commands]
+        end
+      end
+
       describe "Image and service handling" do
         context "when extended docker configuration is used" do
           it "returns image and service when defined" do