Merge branch 'master' into fix/visibility-level-setting-in-forked-projects

* master: (723 commits)
  Bump Rack Attack to v4.3.1 for security fix
  Remove duplicate entry in the changelog
  Remove extra spaces after branchname
  Fix merge-request-reopen button title
  Add branch and tag operation to tree dropdown
  Use gitlab-shell 2.6.9
  Clarify Windows shell executor artifact upload support
  Fix feature specs: we always show the build status if ci_commit is present
  Do not display project group/name when issue and MR are in same project
  Don't create CI status for refs that doesn't have .gitlab-ci.yml, even if the builds are enabled
  Use gitlab-workhorse 0.5.1
  Fix ci_projects migration by using the value only from latest row [ci skip]
  Revert sidebar position for issue and merge request
  Add info on using private Docker registries in CI [ci skip]
  Upgrade Poltergeist to 1.8.1. #4131
  Fix ux issue with "This issue will be closed automatically" message
  Move MR Builds tab next to Commits
  Api support for requesting starred projects for user
  Fix Rubocop complain.
  Fix merge widget JS for buttons
  ...

Conflicts:
	app/models/project.rb

13 files changed, 52 insertions, 362 deletions

diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 07e68216d7f..5c347e432b4 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -30,9 +30,7 @@ module Ci
       helpers Gitlab::CurrentSettings
 
       mount Builds
-      mount Commits
       mount Runners
-      mount Projects
       mount Triggers
     end
   end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 0a586672807..15faa6edd84 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -58,6 +58,7 @@ module Ci
         #   POST /builds/:id/artifacts/authorize
         post ":id/artifacts/authorize" do
           require_gitlab_workhorse!
+          not_allowed! unless Gitlab.config.artifacts.enabled
           build = Ci::Build.find_by_id(params[:id])
           not_found! unless build
           authenticate_build_token!(build)
@@ -91,6 +92,7 @@ module Ci
         #   POST /builds/:id/artifacts
         post ":id/artifacts" do
           require_gitlab_workhorse!
+          not_allowed! unless Gitlab.config.artifacts.enabled
           build = Ci::Build.find_by_id(params[:id])
           not_found! unless build
           authenticate_build_token!(build)
diff --git a/lib/ci/api/commits.rb b/lib/ci/api/commits.rb
deleted file mode 100644
index a60769d8305..00000000000
--- a/lib/ci/api/commits.rb
+++ /dev/null
@@ -1,66 +0,0 @@
-module Ci
-  module API
-    class Commits < Grape::API
-      resource :commits do
-        # Get list of commits per project
-        #
-        # Parameters:
-        #   project_id (required) - The ID of a project
-        #   project_token (requires) - Project token
-        #   page (optional)
-        #   per_page (optional) - items per request (default is 20)
-        #
-        get do
-          required_attributes! [:project_id, :project_token]
-          project = Ci::Project.find(params[:project_id])
-          authenticate_project_token!(project)
-
-          commits = project.commits.page(params[:page]).per(params[:per_page] || 20)
-          present commits, with: Entities::CommitWithBuilds
-        end
-
-        # Create a commit
-        #
-        # Parameters:
-        #   project_id (required) - The ID of a project
-        #   project_token (requires) - Project token
-        #   data (required) - GitLab push data
-        #
-        #   Sample GitLab push data:
-        #   {
-        #     "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
-        #     "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
-        #     "ref": "refs/heads/master",
-        #     "commits": [
-        #       {
-        #         "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
-        #         "message": "Update Catalan translation to e38cb41.",
-        #         "timestamp": "2011-12-12T14:27:31+02:00",
-        #         "url": "http://localhost/diaspora/commits/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
-        #         "author": {
-        #           "name": "Jordi Mallach",
-        #           "email": "jordi@softcatala.org",
-        #         }
-        #       }, .... more commits
-        #     ]
-        #   }
-        #
-        # Example Request:
-        #   POST /commits
-        post do
-          required_attributes! [:project_id, :data, :project_token]
-          project = Ci::Project.find(params[:project_id])
-          authenticate_project_token!(project)
-          commit = Ci::CreateCommitService.new.execute(project, current_user, params[:data])
-
-          if commit.persisted?
-            present commit, with: Entities::CommitWithBuilds
-          else
-            errors = commit.errors.full_messages.join(", ")
-            render_api_error!(errors, 400)
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index 750f421872d..e4ac0545ea2 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -37,15 +37,6 @@ module Ci
         expose :id, :token
       end
 
-      class Project < Grape::Entity
-        expose :id, :name, :token, :default_ref, :gitlab_url, :path,
-          :always_build, :polling_interval, :public, :ssh_url_to_repo, :gitlab_id
-
-        expose :timeout do |model|
-          model.timeout
-        end
-      end
-
       class RunnerProject < Grape::Entity
         expose :id, :project_id, :runner_id
       end
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 02502333756..443563c2e4a 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -6,22 +6,22 @@ module Ci
       UPDATE_RUNNER_EVERY = 60
 
       def authenticate_runners!
-        forbidden! unless params[:token] == GitlabCi::REGISTRATION_TOKEN
+        forbidden! unless runner_registration_token_valid?
       end
 
       def authenticate_runner!
         forbidden! unless current_runner
       end
 
-      def authenticate_project_token!(project)
-        forbidden! unless project.valid_token?(params[:project_token])
-      end
-
       def authenticate_build_token!(build)
         token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
         forbidden! unless token && build.valid_token?(token)
       end
 
+      def runner_registration_token_valid?
+        params[:token] == current_application_settings.ensure_runners_registration_token
+      end
+
       def update_runner_last_contact
         # Use a random threshold to prevent beating DB updates
         contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb
deleted file mode 100644
index d719ad9e8d5..00000000000
--- a/lib/ci/api/projects.rb
+++ /dev/null
@@ -1,195 +0,0 @@
-module Ci
-  module API
-    # Projects API
-    class Projects < Grape::API
-      before { authenticate! }
-
-      resource :projects do
-        # Register new webhook for project
-        #
-        # Parameters
-        #   project_id (required) - The ID of a project
-        #   web_hook (required) - WebHook URL
-        # Example Request
-        #   POST /projects/:project_id/webhooks
-        post ":project_id/webhooks" do
-          required_attributes! [:web_hook]
-
-          project = Ci::Project.find(params[:project_id])
-
-          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
-
-          web_hook = project.web_hooks.new({ url: params[:web_hook] })
-
-          if web_hook.save
-            present web_hook, with: Entities::WebHook
-          else
-            errors = web_hook.errors.full_messages.join(", ")
-            render_api_error!(errors, 400)
-          end
-        end
-
-        # Retrieve all Gitlab CI projects that the user has access to
-        #
-        # Example Request:
-        #   GET /projects
-        get do
-          gitlab_projects = current_user.authorized_projects
-          gitlab_projects = filter_projects(gitlab_projects)
-          gitlab_projects = paginate gitlab_projects
-
-          ids = gitlab_projects.map { |project| project.id }
-
-          projects = Ci::Project.where("gitlab_id IN (?)", ids).load
-          present projects, with: Entities::Project
-        end
-
-        # Retrieve all Gitlab CI projects that the user owns
-        #
-        # Example Request:
-        #   GET /projects/owned
-        get "owned" do
-          gitlab_projects = current_user.owned_projects
-          gitlab_projects = filter_projects(gitlab_projects)
-          gitlab_projects = paginate gitlab_projects
-
-          ids = gitlab_projects.map { |project| project.id }
-
-          projects = Ci::Project.where("gitlab_id IN (?)", ids).load
-          present projects, with: Entities::Project
-        end
-
-        # Retrieve info for a Gitlab CI project
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        # Example Request:
-        #   GET /projects/:id
-        get ":id" do
-          project = Ci::Project.find(params[:id])
-          unauthorized! unless can?(current_user, :read_project, project.gl_project)
-
-          present project, with: Entities::Project
-        end
-
-        # Create Gitlab CI project using Gitlab project info
-        #
-        # Parameters:
-        #   gitlab_id (required)       - The gitlab id of the project
-        #   default_ref                - The branch to run against (defaults to `master`)
-        # Example Request:
-        #   POST /projects
-        post do
-          required_attributes! [:gitlab_id]
-
-          filtered_params = {
-            gitlab_id:       params[:gitlab_id],
-            # we accept gitlab_url for backward compatibility for a while (added to 7.11)
-            default_ref:     params[:default_ref] || 'master'
-          }
-
-          project = Ci::Project.new(filtered_params)
-          project.build_missing_services
-
-          if project.save
-            present project, with: Entities::Project
-          else
-            errors = project.errors.full_messages.join(", ")
-            render_api_error!(errors, 400)
-          end
-        end
-
-        # Update a Gitlab CI project
-        #
-        # Parameters:
-        #   id (required)   - The ID of a project
-        #   default_ref      - The branch to run against (defaults to `master`)
-        # Example Request:
-        #   PUT /projects/:id
-        put ":id" do
-          project = Ci::Project.find(params[:id])
-
-          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
-
-          attrs = attributes_for_keys [:default_ref]
-
-          if project.update_attributes(attrs)
-            present project, with: Entities::Project
-          else
-            errors = project.errors.full_messages.join(", ")
-            render_api_error!(errors, 400)
-          end
-        end
-
-        # Remove a Gitlab CI project
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        # Example Request:
-        #   DELETE /projects/:id
-        delete ":id" do
-          project = Ci::Project.find(params[:id])
-
-          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
-
-          project.destroy
-        end
-
-        # Link a Gitlab CI project to a runner
-        #
-        # Parameters:
-        #   id (required) - The ID of a CI project
-        #   runner_id (required) - The ID of a runner
-        # Example Request:
-        #   POST /projects/:id/runners/:runner_id
-        post ":id/runners/:runner_id" do
-          project = Ci::Project.find(params[:id])
-          runner  = Ci::Runner.find(params[:runner_id])
-
-          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
-
-          options = {
-            project_id: project.id,
-            runner_id:  runner.id
-          }
-
-          runner_project = Ci::RunnerProject.new(options)
-
-          if runner_project.save
-            present runner_project, with: Entities::RunnerProject
-          else
-            errors = project.errors.full_messages.join(", ")
-            render_api_error!(errors, 400)
-          end
-        end
-
-        # Remove a Gitlab CI project from a runner
-        #
-        # Parameters:
-        #   id (required) - The ID of a CI project
-        #   runner_id (required) - The ID of a runner
-        # Example Request:
-        #   DELETE /projects/:id/runners/:runner_id
-        delete ":id/runners/:runner_id" do
-          project = Ci::Project.find(params[:id])
-          runner  = Ci::Runner.find(params[:runner_id])
-
-          unauthorized! unless can?(current_user, :admin_project, project.gl_project)
-
-          options = {
-            project_id: project.id,
-            runner_id:  runner.id
-          }
-
-          runner_project = Ci::RunnerProject.find_by(options)
-
-          if runner_project.present?
-            runner_project.destroy
-          else
-            not_found!
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb
index 1466fe4356e..bfc14fe7a6b 100644
--- a/lib/ci/api/runners.rb
+++ b/lib/ci/api/runners.rb
@@ -3,17 +3,6 @@ module Ci
     # Runners API
     class Runners < Grape::API
       resource :runners do
-        # Get list of all available runners
-        #
-        # Example Request:
-        #   GET /runners
-        get do
-          authenticate!
-          runners = Ci::Runner.all
-
-          present runners, with: Entities::Runner
-        end
-
         # Delete runner
         # Parameters:
         #   token (required) - The unique token of runner
@@ -40,14 +29,14 @@ module Ci
           required_attributes! [:token]
 
           runner =
-            if params[:token] == GitlabCi::REGISTRATION_TOKEN
+            if runner_registration_token_valid?
               # Create shared runner. Requires admin access
               Ci::Runner.create(
                 description: params[:description],
                 tag_list: params[:tag_list],
                 is_shared: true
               )
-            elsif project = Ci::Project.find_by(token: params[:token])
+            elsif project = Project.find_by(runners_token: params[:token])
               # Create a specific runner for project.
               project.runners.create(
                 description: params[:description],
diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb
index 40907d6db54..63b42113513 100644
--- a/lib/ci/api/triggers.rb
+++ b/lib/ci/api/triggers.rb
@@ -14,7 +14,7 @@ module Ci
         post ":id/refs/:ref/trigger" do
           required_attributes! [:token]
 
-          project = Ci::Project.find(params[:id])
+          project = Project.find_by(ci_id: params[:id].to_i)
           trigger = Ci::Trigger.find_by_token(params[:token].to_s)
           not_found! unless project && trigger
           unauthorized! unless trigger.project == project
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index 5ff7407c6fe..d53bdcbd0f2 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -60,7 +60,7 @@ module Ci
 
     class BuildTime < Chart
       def collect
-        commits = project.commits.last(30)
+        commits = project.ci_commits.last(30)
 
         commits.each do |commit|
           @labels << commit.short_sha
diff --git a/lib/ci/current_settings.rb b/lib/ci/current_settings.rb
deleted file mode 100644
index fd78b024970..00000000000
--- a/lib/ci/current_settings.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-module Ci
-  module CurrentSettings
-    def current_application_settings
-      key = :ci_current_application_settings
-
-      RequestStore.store[key] ||= begin
-        if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('ci_application_settings')
-          Ci::ApplicationSetting.current || Ci::ApplicationSetting.create_from_defaults
-        else
-          fake_application_settings
-        end
-      end
-    end
-
-    def fake_application_settings
-      OpenStruct.new(
-        all_broken_builds: Ci::Settings.gitlab_ci['all_broken_builds'],
-        add_pusher: Ci::Settings.gitlab_ci['add_pusher'],
-      )
-    end
-  end
-end
diff --git a/lib/ci/git.rb b/lib/ci/git.rb
deleted file mode 100644
index 7acc3f38edb..00000000000
--- a/lib/ci/git.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-module Ci
-  module Git
-    BLANK_SHA = '0' * 40
-  end
-end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 3beafcad117..bcdfd38d292 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -10,7 +10,7 @@ module Ci
     attr_reader :before_script, :image, :services, :variables, :path, :cache
 
     def initialize(config, path = nil)
-      @config = YAML.load(config)
+      @config = YAML.safe_load(config, [Symbol])
       @path = path
 
       unless @config.is_a? Hash
@@ -132,26 +132,36 @@ module Ci
     end
 
     def validate_job!(name, job)
+      validate_job_name!(name)
+      validate_job_keys!(name, job)
+      validate_job_types!(name, job)
+
+      validate_job_stage!(name, job) if job[:stage]
+      validate_job_cache!(name, job) if job[:cache]
+      validate_job_artifacts!(name, job) if job[:artifacts]
+    end
+
+    private
+
+    def validate_job_name!(name)
       if name.blank? || !validate_string(name)
         raise ValidationError, "job name should be non-empty string"
       end
+    end
 
+    def validate_job_keys!(name, job)
       job.keys.each do |key|
         unless ALLOWED_JOB_KEYS.include? key
           raise ValidationError, "#{name} job: unknown parameter #{key}"
         end
       end
+    end
 
+    def validate_job_types!(name, job)
       if !validate_string(job[:script]) && !validate_array_of_strings(job[:script])
         raise ValidationError, "#{name} job: script should be a string or an array of a strings"
       end
 
-      if job[:stage]
-        unless job[:stage].is_a?(String) && job[:stage].in?(stages)
-          raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}"
-        end
-      end
-
       if job[:image] && !validate_string(job[:image])
         raise ValidationError, "#{name} job: image should be a string"
       end
@@ -172,36 +182,40 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
-      if job[:cache]
-        if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
-          raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
-        end
-
-        if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths])
-          raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings"
-        end
+      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
+        raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
 
-      if job[:artifacts]
-        if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
-          raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
-        end
+      if job[:when] && !job[:when].in?(%w(on_success on_failure always))
+        raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always"
+      end
+    end
 
-        if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
-          raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
-        end
+    def validate_job_stage!(name, job)
+      unless job[:stage].is_a?(String) && job[:stage].in?(stages)
+        raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}"
       end
+    end
 
-      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
-        raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
+    def validate_job_cache!(name, job)
+      if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
+        raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
       end
 
-      if job[:when] && !job[:when].in?(%w(on_success on_failure always))
-        raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always"
+      if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths])
+        raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings"
       end
     end
 
-    private
+    def validate_job_artifacts!(name, job)
+      if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
+        raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
+      end
+
+      if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
+        raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
+      end
+    end
 
     def validate_array_of_strings(values)
       values.is_a?(Array) && values.all? { |value| validate_string(value) }
diff --git a/lib/ci/scheduler.rb b/lib/ci/scheduler.rb
deleted file mode 100644
index ee0958f4be1..00000000000
--- a/lib/ci/scheduler.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-module Ci
-  class Scheduler
-    def perform
-      projects = Ci::Project.where(always_build: true).all
-      projects.each do |project|
-        last_commit = project.commits.last
-        next unless last_commit && last_commit.last_build
-
-        interval = project.polling_interval
-        if (last_commit.last_build.created_at + interval.hours) < Time.now
-          last_commit.retry
-        end
-      end
-    end
-  end
-end