6 files changed, 112 insertions, 50 deletions

diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb
index ac6d667cf8d..229050151d3 100644
--- a/lib/ci/ansi2html.rb
+++ b/lib/ci/ansi2html.rb
@@ -23,8 +23,8 @@ module Ci
       cross:      0x10,
     }
 
-    def self.convert(ansi)
-      Converter.new().convert(ansi)
+    def self.convert(ansi, state = nil)
+      Converter.new.convert(ansi, state)
     end
 
     class Converter
@@ -84,22 +84,38 @@ module Ci
       def on_107(s) set_bg_color(7, 'l') end
       def on_109(s) set_bg_color(9, 'l') end
 
-      def convert(ansi)
-        @out = ""
-        @n_open_tags = 0
-        reset()
+      attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask
+
+      STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask]
+
+      def convert(raw, new_state)
+        reset_state
+        restore_state(raw, new_state) if new_state.present?
+
+        start = @offset
+        ansi = raw[@offset..-1]
+
+        open_new_tag
 
-        s = StringScanner.new(ansi.gsub("<", "&lt;"))
-        while(!s.eos?)
+        s = StringScanner.new(ansi)
+        until s.eos?
           if s.scan(/\e([@-_])(.*?)([@-~])/)
             handle_sequence(s)
+          elsif s.scan(/\e(([@-_])(.*?)?)?$/)
+            break
+          elsif s.scan(/</)
+            @out << '&lt;'
+          elsif s.scan(/\n/)
+            @out << '<br>'
           else
             @out << s.scan(/./m)
           end
+          @offset += s.matched_size
         end
 
         close_open_tags()
-        @out
+
+        { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 }
       end
 
       def handle_sequence(s)
@@ -121,6 +137,20 @@ module Ci
 
         evaluate_command_stack(commands)
 
+        open_new_tag
+      end
+
+      def evaluate_command_stack(stack)
+        return unless command = stack.shift()
+
+        if self.respond_to?("on_#{command}", true)
+          self.send("on_#{command}", stack)
+        end
+
+        evaluate_command_stack(stack)
+      end
+
+      def open_new_tag
         css_classes = []
 
         unless @fg_color.nil?
@@ -138,20 +168,8 @@ module Ci
           css_classes << "term-#{css_class}" if @style_mask & flag != 0
         end
 
-        open_new_tag(css_classes) if css_classes.length > 0
-      end
+        return if css_classes.empty?
 
-      def evaluate_command_stack(stack)
-        return unless command = stack.shift()
-
-        if self.respond_to?("on_#{command}", true)
-          self.send("on_#{command}", stack)
-        end
-
-        evaluate_command_stack(stack)
-      end
-
-      def open_new_tag(css_classes)
         @out << %{<span class="#{css_classes.join(' ')}">}
         @n_open_tags += 1
       end
@@ -163,6 +181,31 @@ module Ci
         end
       end
 
+      def reset_state
+        @offset = 0
+        @n_open_tags = 0
+        @out = ''
+        reset
+      end
+
+      def state
+        state = STATE_PARAMS.inject({}) do |h, param|
+          h[param] = send(param)
+          h
+        end
+        Base64.urlsafe_encode64(state.to_json)
+      end
+
+      def restore_state(raw, new_state)
+        state = Base64.urlsafe_decode64(new_state)
+        state = JSON.parse(state, symbolize_names: true)
+        return if state[:offset].to_i > raw.length
+
+        STATE_PARAMS.each do |param|
+          send("#{param}=".to_sym, state[param])
+        end
+      end
+
       def reset
         @fg_color = nil
         @bg_color = nil
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 353c4ddebf8..17bb99a2ae5 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -1,9 +1,7 @@
-Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file}
-
 module Ci
   module API
     class API < Grape::API
-      include APIGuard
+      include ::API::APIGuard
       version 'v1', using: :path
 
       rescue_from ActiveRecord::RecordNotFound do
@@ -31,9 +29,9 @@ module Ci
       helpers ::API::Helpers
       helpers Gitlab::CurrentSettings
 
-      mount Builds
-      mount Runners
-      mount Triggers
+      mount ::Ci::API::Builds
+      mount ::Ci::API::Runners
+      mount ::Ci::API::Triggers
     end
   end
 end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b25e0e573a8..a902ced35d7 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -56,7 +56,7 @@ module Ci
 
       class TriggerRequest < Grape::Entity
         expose :id, :variables
-        expose :commit, using: Commit
+        expose :pipeline, using: Commit, as: :commit
       end
     end
   end
diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb
index 192b1d18a51..0c41f22c7c5 100644
--- a/lib/ci/api/runners.rb
+++ b/lib/ci/api/runners.rb
@@ -28,20 +28,20 @@ module Ci
         post "register" do
           required_attributes! [:token]
 
+          attributes = { description: params[:description],
+                         tag_list: params[:tag_list] }
+
+          unless params[:run_untagged].nil?
+            attributes[:run_untagged] = params[:run_untagged]
+          end
+
           runner =
             if runner_registration_token_valid?
               # Create shared runner. Requires admin access
-              Ci::Runner.create(
-                description: params[:description],
-                tag_list: params[:tag_list],
-                is_shared: true
-              )
+              Ci::Runner.create(attributes.merge(is_shared: true))
             elsif project = Project.find_by(runners_token: params[:token])
               # Create a specific runner for project.
-              project.runners.create(
-                description: params[:description],
-                tag_list: params[:tag_list]
-              )
+              project.runners.create(attributes)
             end
 
           return forbidden! unless runner
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index d53bdcbd0f2..5270108ef0f 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -60,11 +60,12 @@ module Ci
 
     class BuildTime < Chart
       def collect
-        commits = project.ci_commits.last(30)
+        commits = project.pipelines.last(30)
 
         commits.each do |commit|
           @labels << commit.short_sha
-          @build_times << (commit.duration / 60)
+          duration = commit.duration || 0
+          @build_times << (duration / 60)
         end
       end
     end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 504d3df9d34..40a5d180fd0 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -1,6 +1,6 @@
 module Ci
   class GitlabCiYamlProcessor
-    class ValidationError < StandardError;end
+    class ValidationError < StandardError; end
 
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
@@ -8,22 +8,20 @@ module Ci
     ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
                         :allow_failure, :type, :stage, :when, :artifacts, :cache,
                         :dependencies, :before_script, :after_script, :variables]
+    ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
+    ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when]
 
     attr_reader :before_script, :after_script, :image, :services, :path, :cache
 
     def initialize(config, path = nil)
-      @config = YAML.safe_load(config, [Symbol], [], true)
+      @config = Gitlab::Ci::Config.new(config).to_hash
       @path = path
 
-      unless @config.is_a? Hash
-        raise ValidationError, "YAML should be a hash"
-      end
-
-      @config = @config.deep_symbolize_keys
-
       initial_parsing
 
       validate!
+    rescue Gitlab::Ci::Config::Loader::FormatError => e
+      raise ValidationError, e.message
     end
 
     def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
@@ -139,6 +137,12 @@ module Ci
     end
 
     def validate_global_cache!
+      @cache.keys.each do |key|
+        unless ALLOWED_CACHE_KEYS.include? key
+          raise ValidationError, "#{name} cache unknown parameter #{key}"
+        end
+      end
+
       if @cache[:key] && !validate_string(@cache[:key])
         raise ValidationError, "cache:key parameter should be a string"
       end
@@ -204,7 +208,7 @@ module Ci
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
 
-      if job[:when] && !job[:when].in?(%w(on_success on_failure always))
+      if job[:when] && !job[:when].in?(%w[on_success on_failure always])
         raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always"
       end
     end
@@ -237,6 +241,12 @@ module Ci
     end
 
     def validate_job_cache!(name, job)
+      job[:cache].keys.each do |key|
+        unless ALLOWED_CACHE_KEYS.include? key
+          raise ValidationError, "#{name} job: cache unknown parameter #{key}"
+        end
+      end
+
       if job[:cache][:key] && !validate_string(job[:cache][:key])
         raise ValidationError, "#{name} job: cache:key parameter should be a string"
       end
@@ -251,6 +261,12 @@ module Ci
     end
 
     def validate_job_artifacts!(name, job)
+      job[:artifacts].keys.each do |key|
+        unless ALLOWED_ARTIFACTS_KEYS.include? key
+          raise ValidationError, "#{name} job: artifacts unknown parameter #{key}"
+        end
+      end
+
       if job[:artifacts][:name] && !validate_string(job[:artifacts][:name])
         raise ValidationError, "#{name} job: artifacts:name parameter should be a string"
       end
@@ -262,10 +278,14 @@ module Ci
       if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
         raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
       end
+
+      if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always])
+        raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always"
+      end
     end
 
     def validate_job_dependencies!(name, job)
-      if !validate_array_of_strings(job[:dependencies])
+      unless validate_array_of_strings(job[:dependencies])
         raise ValidationError, "#{name} job: dependencies parameter should be an array of strings"
       end