Merge branch 'master' into james11/gitlab-ce-removable-group-owner

6 files changed, 199 insertions, 36 deletions

diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 0a4cbf69b63..07e68216d7f 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -27,6 +27,7 @@ module Ci
 
       helpers Helpers
       helpers ::API::Helpers
+      helpers Gitlab::CurrentSettings
 
       mount Builds
       mount Commits
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 83ca1e6481c..0a586672807 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -47,6 +47,106 @@ module Ci
             build.drop
           end
         end
+
+        # Authorize artifacts uploading for build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        #   filesize (optional) - the size of uploaded file
+        # Example Request:
+        #   POST /builds/:id/artifacts/authorize
+        post ":id/artifacts/authorize" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          if params[:filesize]
+            file_size = params[:filesize].to_i
+            file_to_large! unless file_size < max_artifacts_size
+          end
+
+          status 200
+          { TempPath: ArtifactUploader.artifacts_upload_path }
+        end
+
+        # Upload artifacts to build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        #   file (required) - The uploaded file
+        # Parameters (accelerated by GitLab Workhorse):
+        #   file.path - path to locally stored body (generated by Workhorse)
+        #   file.name - real filename as send in Content-Disposition
+        #   file.type - real content type as send in Content-Type
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Body:
+        #   The file content
+        #
+        # Example Request:
+        #   POST /builds/:id/artifacts
+        post ":id/artifacts" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
+          file_to_large! unless file.size < max_artifacts_size
+
+          if build.update_attributes(artifacts_file: file)
+            present build, with: Entities::Build
+          else
+            render_validation_error!(build)
+          end
+        end
+
+        # Download the artifacts file from build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   GET /builds/:id/artifacts
+        get ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          artifacts_file = build.artifacts_file
+
+          unless artifacts_file.file_storage?
+            return redirect_to build.artifacts_file.url
+          end
+
+          unless artifacts_file.exists?
+            not_found!
+          end
+
+          present_file!(artifacts_file.path, artifacts_file.filename)
+        end
+
+        # Remove the artifacts file from build
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   DELETE /builds/:id/artifacts
+        delete ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          build.remove_artifacts_file!
+        end
       end
     end
   end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b80c0b8b273..750f421872d 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -11,10 +11,16 @@ module Ci
         expose :builds
       end
 
+      class ArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
       class Build < Grape::Entity
         expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
           :before_sha, :allow_git_fetch, :project_name
 
+        expose :name, :token, :stage
+
         expose :options do |model|
           model.options
         end
@@ -24,6 +30,7 @@ module Ci
         end
 
         expose :variables
+        expose :artifacts_file, using: ArtifactFile
       end
 
       class Runner < Grape::Entity
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index e602cda81d6..02502333756 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -1,6 +1,8 @@
 module Ci
   module API
     module Helpers
+      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+      BUILD_TOKEN_PARAM = :token
       UPDATE_RUNNER_EVERY = 60
 
       def authenticate_runners!
@@ -15,8 +17,15 @@ module Ci
         forbidden! unless project.valid_token?(params[:project_token])
       end
 
+      def authenticate_build_token!(build)
+        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+        forbidden! unless token && build.valid_token?(token)
+      end
+
       def update_runner_last_contact
-        if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY
+        # Use a random threshold to prevent beating DB updates
+        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
+        if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age
           current_runner.update_attributes(contacted_at: Time.now)
         end
       end
@@ -30,6 +39,10 @@ module Ci
         info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
         current_runner.update(info)
       end
+
+      def max_artifacts_size
+        current_application_settings.max_artifacts_size.megabytes.to_i
+      end
     end
   end
 end
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index 915a4f526a6..5ff7407c6fe 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -60,7 +60,8 @@ module Ci
 
     class BuildTime < Chart
       def collect
-        commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30)
+        commits = project.commits.last(30)
+
         commits.each do |commit|
           @labels << commit.short_sha
           @build_times << (commit.duration / 60)
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index efcd2faffc7..3beafcad117 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -4,13 +4,14 @@ module Ci
 
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
-    ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
+    ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache]
+    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache]
 
-    attr_reader :before_script, :image, :services, :variables
+    attr_reader :before_script, :image, :services, :variables, :path, :cache
 
-    def initialize(config)
+    def initialize(config, path = nil)
       @config = YAML.load(config)
+      @path = path
 
       unless @config.is_a? Hash
         raise ValidationError, "YAML should be a hash"
@@ -45,6 +46,7 @@ module Ci
       @services = @config[:services]
       @stages = @config[:stages] || @config[:types]
       @variables = @config[:variables] || {}
+      @cache = @config[:cache]
       @config.except!(*ALLOWED_YAML_KEYS)
 
       # anything that doesn't have script is considered as unknown
@@ -63,26 +65,6 @@ module Ci
       end
     end
 
-    def process?(only_params, except_params, ref, tag)
-      return true if only_params.nil? && except_params.nil?
-
-      if only_params
-        return true if tag && only_params.include?("tags")
-        return true if !tag && only_params.include?("branches")
-        
-        only_params.find do |pattern|
-          match_ref?(pattern, ref)
-        end
-      else
-        return false if tag && except_params.include?("tags")
-        return false if !tag && except_params.include?("branches")
-
-        except_params.each do |pattern|
-          return false if match_ref?(pattern, ref)
-        end
-      end
-    end
-
     def build_job(name, job)
       {
         stage_idx: stages.index(job[:stage]),
@@ -96,19 +78,13 @@ module Ci
         when: job[:when] || 'on_success',
         options: {
           image: job[:image] || @image,
-          services: job[:services] || @services
+          services: job[:services] || @services,
+          artifacts: job[:artifacts],
+          cache: job[:cache] || @cache,
         }.compact
       }
     end
 
-    def match_ref?(pattern, ref)
-      if pattern.first == "/" && pattern.last == "/"
-        Regexp.new(pattern[1...-1]) =~ ref
-      else
-        pattern == ref
-      end
-    end
-
     def normalize_script(script)
       if script.is_a? Array
         script.join("\n")
@@ -138,6 +114,16 @@ module Ci
         raise ValidationError, "variables should be a map of key-valued strings"
       end
 
+      if @cache
+        if @cache[:untracked] && !validate_boolean(@cache[:untracked])
+          raise ValidationError, "cache:untracked parameter should be an boolean"
+        end
+
+        if @cache[:paths] && !validate_array_of_strings(@cache[:paths])
+          raise ValidationError, "cache:paths parameter should be an array of strings"
+        end
+      end
+
       @jobs.each do |name, job|
         validate_job!(name, job)
       end
@@ -186,7 +172,27 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
-      if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+      if job[:cache]
+        if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
+          raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
+        end
+
+        if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths])
+          raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings"
+        end
+      end
+
+      if job[:artifacts]
+        if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
+          raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
+        end
+
+        if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
+          raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
+        end
+      end
+
+      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
 
@@ -208,5 +214,40 @@ module Ci
     def validate_string(value)
       value.is_a?(String) || value.is_a?(Symbol)
     end
+
+    def validate_boolean(value)
+      value.in?([true, false])
+    end
+
+    def process?(only_params, except_params, ref, tag)
+      if only_params.present?
+        return false unless matching?(only_params, ref, tag)
+      end
+
+      if except_params.present?
+        return false if matching?(except_params, ref, tag)
+      end
+
+      true
+    end
+
+    def matching?(patterns, ref, tag)
+      patterns.any? do |pattern|
+        match_ref?(pattern, ref, tag)
+      end
+    end
+
+    def match_ref?(pattern, ref, tag)
+      pattern, path = pattern.split('@', 2)
+      return false if path && path != self.path
+      return true if tag && pattern == 'tags'
+      return true if !tag && pattern == 'branches'
+
+      if pattern.first == "/" && pattern.last == "/"
+        Regexp.new(pattern[1...-1]) =~ ref
+      else
+        pattern == ref
+      end
+    end
   end
 end