Merge branch 'master' into 'email-in-slash-commands'email-in-slash-commands

# Conflicts:
#   doc/user/project/slash_commands.md
#   doc/workflow/README.md

13 files changed, 300 insertions, 55 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 6b8bfbbdae6..e14464c1b0d 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -18,27 +18,20 @@ module API
     end
 
     rescue_from :all do |exception|
-      # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
-      # why is this not wrapped in something reusable?
-      trace = exception.backtrace
-
-      message = "\n#{exception.class} (#{exception.message}):\n"
-      message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code)
-      message << "  " << trace.join("\n  ")
-
-      API.logger.add Logger::FATAL, message
-      rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
+      handle_api_exception(exception)
     end
 
     format :json
     content_type :txt, "text/plain"
 
     # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
+    helpers ::SentryHelper
     helpers ::API::Helpers
 
     mount ::API::AccessRequests
     mount ::API::AwardEmoji
     mount ::API::Branches
+    mount ::API::BroadcastMessages
     mount ::API::Builds
     mount ::API::CommitStatuses
     mount ::API::Commits
@@ -75,5 +68,6 @@ module API
     mount ::API::Triggers
     mount ::API::Users
     mount ::API::Variables
+    mount ::API::MergeRequestDiffs
   end
 end
diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
index 2efe7e3adf3..7c22b17e4e5 100644
--- a/lib/api/award_emoji.rb
+++ b/lib/api/award_emoji.rb
@@ -54,7 +54,7 @@ module API
           post endpoint do
             required_attributes! [:name]
 
-            not_found!('Award Emoji') unless can_read_awardable?
+            not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable?
 
             award = awardable.create_award_emoji(params[:name], current_user)
 
@@ -92,6 +92,10 @@ module API
         can?(current_user, ability, awardable)
       end
 
+      def can_award_awardable?
+        awardable.user_can_award?(current_user, params[:name])
+      end
+
       def awardable
         @awardable ||=
           begin
diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb
new file mode 100644
index 00000000000..fb2a4148011
--- /dev/null
+++ b/lib/api/broadcast_messages.rb
@@ -0,0 +1,99 @@
+module API
+  class BroadcastMessages < Grape::API
+    before { authenticate! }
+    before { authenticated_as_admin! }
+
+    resource :broadcast_messages do
+      helpers do
+        def find_message
+          BroadcastMessage.find(params[:id])
+        end
+      end
+
+      desc 'Get all broadcast messages' do
+        detail 'This feature was introduced in GitLab 8.12.'
+        success Entities::BroadcastMessage
+      end
+      params do
+        optional :page,     type: Integer, desc: 'Current page number'
+        optional :per_page, type: Integer, desc: 'Number of messages per page'
+      end
+      get do
+        messages = BroadcastMessage.all
+
+        present paginate(messages), with: Entities::BroadcastMessage
+      end
+
+      desc 'Create a broadcast message' do
+        detail 'This feature was introduced in GitLab 8.12.'
+        success Entities::BroadcastMessage
+      end
+      params do
+        requires :message,   type: String,   desc: 'Message to display'
+        optional :starts_at, type: DateTime, desc: 'Starting time', default: -> { Time.zone.now }
+        optional :ends_at,   type: DateTime, desc: 'Ending time',   default: -> { 1.hour.from_now }
+        optional :color,     type: String,   desc: 'Background color'
+        optional :font,      type: String,   desc: 'Foreground color'
+      end
+      post do
+        create_params = declared(params, include_missing: false).to_h
+        message = BroadcastMessage.create(create_params)
+
+        if message.persisted?
+          present message, with: Entities::BroadcastMessage
+        else
+          render_validation_error!(message)
+        end
+      end
+
+      desc 'Get a specific broadcast message' do
+        detail 'This feature was introduced in GitLab 8.12.'
+        success Entities::BroadcastMessage
+      end
+      params do
+        requires :id, type: Integer, desc: 'Broadcast message ID'
+      end
+      get ':id' do
+        message = find_message
+
+        present message, with: Entities::BroadcastMessage
+      end
+
+      desc 'Update a broadcast message' do
+        detail 'This feature was introduced in GitLab 8.12.'
+        success Entities::BroadcastMessage
+      end
+      params do
+        requires :id,        type: Integer,  desc: 'Broadcast message ID'
+        optional :message,   type: String,   desc: 'Message to display'
+        optional :starts_at, type: DateTime, desc: 'Starting time'
+        optional :ends_at,   type: DateTime, desc: 'Ending time'
+        optional :color,     type: String,   desc: 'Background color'
+        optional :font,      type: String,   desc: 'Foreground color'
+      end
+      put ':id' do
+        message = find_message
+        update_params = declared(params, include_missing: false).to_h
+
+        if message.update(update_params)
+          present message, with: Entities::BroadcastMessage
+        else
+          render_validation_error!(message)
+        end
+      end
+
+      desc 'Delete a broadcast message' do
+        detail 'This feature was introduced in GitLab 8.12.'
+        success Entities::BroadcastMessage
+      end
+      params do
+        requires :id, type: Integer, desc: 'Broadcast message ID'
+      end
+      delete ':id' do
+        message = find_message
+
+        present message.destroy, with: Entities::BroadcastMessage
+      end
+    end
+  end
+end
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 4df6ca8333e..5e3c9563703 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -64,7 +64,7 @@ module API
           ref = branches.first
         end
 
-        pipeline = @project.ensure_pipeline(commit.sha, ref, current_user)
+        pipeline = @project.ensure_pipeline(ref, commit.sha, current_user)
 
         name = params[:name] || params[:context]
         status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref])
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 54ce2dcfa57..3faba79415b 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -49,7 +49,7 @@ module API
     class ProjectHook < Hook
       expose :project_id, :push_events
       expose :issues_events, :merge_requests_events, :tag_push_events
-      expose :note_events, :build_events, :pipeline_events
+      expose :note_events, :build_events, :pipeline_events, :wiki_page_events
       expose :enable_ssl_verification
     end
 
@@ -76,20 +76,29 @@ module API
       expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group }
       expose :name, :name_with_namespace
       expose :path, :path_with_namespace
-      expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled
+      expose :container_registry_enabled
+
+      # Expose old field names with the new permissions methods to keep API compatible
+      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) }
+      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) }
+      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) }
+      expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) }
+      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) }
+
       expose :created_at, :last_activity_at
-      expose :shared_runners_enabled
+      expose :shared_runners_enabled, :lfs_enabled
       expose :creator_id
       expose :namespace
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
       expose :avatar_url
       expose :star_count, :forks_count
-      expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? }
+      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds
       expose :shared_with_groups do |project, options|
         SharedGroup.represent(project.project_group_links.all, options)
       end
+      expose :only_allow_merge_if_build_succeeds
     end
 
     class Member < UserBasic
@@ -177,6 +186,10 @@ module API
 
       # TODO (rspeicher): Deprecated; remove in 9.0
       expose(:expires_at) { |snippet| nil }
+
+      expose :web_url do |snippet, options|
+        Gitlab::UrlBuilder.build(snippet)
+      end
     end
 
     class ProjectEntity < Grape::Entity
@@ -206,6 +219,11 @@ module API
       expose :user_notes_count
       expose :upvotes, :downvotes
       expose :due_date
+      expose :confidential
+
+      expose :web_url do |issue, options|
+        Gitlab::UrlBuilder.build(issue)
+      end
     end
 
     class ExternalIssue < Grape::Entity
@@ -223,12 +241,18 @@ module API
       expose :milestone, using: Entities::Milestone
       expose :merge_when_build_succeeds
       expose :merge_status
+      expose :diff_head_sha, as: :sha
+      expose :merge_commit_sha
       expose :subscribed do |merge_request, options|
         merge_request.subscribed?(options[:current_user])
       end
       expose :user_notes_count
       expose :should_remove_source_branch?, as: :should_remove_source_branch
       expose :force_remove_source_branch?, as: :force_remove_source_branch
+
+      expose :web_url do |merge_request, options|
+        Gitlab::UrlBuilder.build(merge_request)
+      end
     end
 
     class MergeRequestChanges < MergeRequest
@@ -237,6 +261,19 @@ module API
       end
     end
 
+    class MergeRequestDiff < Grape::Entity
+      expose :id, :head_commit_sha, :base_commit_sha, :start_commit_sha,
+        :created_at, :merge_request_id, :state, :real_size
+    end
+
+    class MergeRequestDiffFull < MergeRequestDiff
+      expose :commits, using: Entities::RepoCommit
+
+      expose :diffs, using: Entities::RepoDiff do |compare, _|
+        compare.raw_diffs(all_diffs: true).to_a
+      end
+    end
+
     class SSHKey < Grape::Entity
       expose :id, :title, :key, :created_at
     end
@@ -514,13 +551,12 @@ module API
       expose :duration
     end
 
-    class Environment < Grape::Entity
+    class EnvironmentBasic < Grape::Entity
       expose :id, :name, :external_url
-      expose :project, using: Entities::Project
     end
 
-    class EnvironmentBasic < Grape::Entity
-      expose :id, :name, :external_url
+    class Environment < EnvironmentBasic
+      expose :project, using: Entities::Project
     end
 
     class Deployment < Grape::Entity
@@ -549,5 +585,10 @@ module API
     class Template < Grape::Entity
       expose :name, :content
     end
+
+    class BroadcastMessage < Grape::Entity
+      expose :id, :message, :starts_at, :ends_at, :color, :font
+      expose :active?, as: :active
+    end
   end
 end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 9d8b8d737a9..d2df77238d5 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -30,7 +30,7 @@ module API
       # Example Request:
       #   POST /groups
       post do
-        authorize! :create_group, current_user
+        authorize! :create_group
         required_attributes! [:name, :path]
 
         attrs = attributes_for_keys [:name, :path, :description, :visibility_level]
@@ -97,7 +97,7 @@ module API
         group = find_group(params[:id])
         projects = GroupProjectsFinder.new(group).execute(current_user)
         projects = paginate projects
-        present projects, with: Entities::Project
+        present projects, with: Entities::Project, user: current_user
       end
 
       # Transfer a project to the Group namespace
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index d0469d6602d..6a20ba95a79 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -129,7 +129,7 @@ module API
       forbidden! unless current_user.is_admin?
     end
 
-    def authorize!(action, subject)
+    def authorize!(action, subject = nil)
       forbidden! unless can?(current_user, action, subject)
     end
 
@@ -148,7 +148,7 @@ module API
     end
 
     def can?(object, action, subject)
-      abilities.allowed?(object, action, subject)
+      Ability.allowed?(object, action, subject)
     end
 
     # Checks the occurrences of required attributes, each attribute must be present in the params hash
@@ -279,6 +279,24 @@ module API
       error!({ 'message' => message }, status)
     end
 
+    def handle_api_exception(exception)
+      if sentry_enabled? && report_exception?(exception)
+        define_params_for_grape_middleware
+        sentry_context
+        Raven.capture_exception(exception)
+      end
+
+      # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60
+      trace = exception.backtrace
+
+      message = "\n#{exception.class} (#{exception.message}):\n"
+      message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code)
+      message << "  " << trace.join("\n  ")
+
+      API.logger.add Logger::FATAL, message
+      rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
+    end
+
     # Projects helpers
 
     def filter_projects(projects)
@@ -390,14 +408,6 @@ module API
       links.join(', ')
     end
 
-    def abilities
-      @abilities ||= begin
-                       abilities = Six.new
-                       abilities << Ability
-                       abilities
-                     end
-    end
-
     def secret_token
       File.read(Gitlab.config.gitlab_shell.secret_file).chomp
     end
@@ -419,5 +429,19 @@ module API
         Entities::Issue
       end
     end
+
+    # The Grape Error Middleware only has access to env but no params. We workaround this by
+    # defining a method that returns the right value.
+    def define_params_for_grape_middleware
+      self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys }
+    end
+
+    # We could get a Grape or a standard Ruby exception. We should only report anything that
+    # is clearly an error.
+    def report_exception?(exception)
+      return true unless exception.respond_to?(:status)
+
+      exception.status == 500
+    end
   end
 end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index d8e9ac406c4..6e6efece7c4 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -101,6 +101,35 @@ module API
           {}
         end
       end
+
+      post '/two_factor_recovery_codes' do
+        status 200
+
+        key = Key.find_by(id: params[:key_id])
+
+        unless key
+          return { 'success' => false, 'message' => 'Could not find the given key' }
+        end
+
+        if key.is_a?(DeployKey)
+          return { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' }
+        end
+
+        user = key.user
+
+        unless user
+          return { success: false, message: 'Could not find a user for the given key' }
+        end
+
+        unless user.two_factor_enabled?
+          return { success: false, message: 'Two-factor authentication is not enabled for this user' }
+        end
+
+        codes = user.generate_otp_backup_codes!
+        user.save!
+
+        { success: true, recovery_codes: codes }
+      end
     end
   end
 end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 077258faee1..556684187d8 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -140,12 +140,13 @@ module API
       #   labels (optional)       - The labels of an issue
       #   created_at (optional)   - Date time string, ISO 8601 formatted
       #   due_date (optional)     - Date time string in the format YEAR-MONTH-DAY
+      #   confidential (optional) - Boolean parameter if the issue should be confidential
       # Example Request:
       #   POST /projects/:id/issues
       post ':id/issues' do
         required_attributes! [:title]
 
-        keys = [:title, :description, :assignee_id, :milestone_id, :due_date]
+        keys = [:title, :description, :assignee_id, :milestone_id, :due_date, :confidential]
         keys << :created_at if current_user.admin? || user_project.owner == current_user
         attrs = attributes_for_keys(keys)
 
@@ -154,21 +155,19 @@ module API
           render_api_error!({ labels: errors }, 400)
         end
 
-        project = user_project
+        attrs[:labels] = params[:labels] if params[:labels]
 
-        issue = ::Issues::CreateService.new(project, current_user, attrs.merge(request: request, api: true)).execute
+        # Convert and filter out invalid confidential flags
+        attrs['confidential'] = to_boolean(attrs['confidential'])
+        attrs.delete('confidential') if attrs['confidential'].nil?
+
+        issue = ::Issues::CreateService.new(user_project, current_user, attrs.merge(request: request, api: true)).execute
 
         if issue.spam?
           render_api_error!({ error: 'Spam detected' }, 400)
         end
 
         if issue.valid?
-          # Find or create labels and attach to issue. Labels are valid because
-          # we already checked its name, so there can't be an error here
-          if params[:labels].present?
-            issue.add_labels_by_names(params[:labels].split(','))
-          end
-
           present issue, with: Entities::Issue, current_user: current_user
         else
           render_validation_error!(issue)
@@ -188,12 +187,13 @@ module API
       #   state_event (optional) - The state event of an issue (close|reopen)
       #   updated_at (optional) - Date time string, ISO 8601 formatted
       #   due_date (optional)     - Date time string in the format YEAR-MONTH-DAY
+      #   confidential (optional) - Boolean parameter if the issue should be confidential
       # Example Request:
       #   PUT /projects/:id/issues/:issue_id
       put ':id/issues/:issue_id' do
         issue = user_project.issues.find(params[:issue_id])
         authorize! :update_issue, issue
-        keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date]
+        keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date, :confidential]
         keys << :updated_at if current_user.admin? || user_project.owner == current_user
         attrs = attributes_for_keys(keys)
 
@@ -202,17 +202,15 @@ module API
           render_api_error!({ labels: errors }, 400)
         end
 
+        attrs[:labels] = params[:labels] if params[:labels]
+
+        # Convert and filter out invalid confidential flags
+        attrs['confidential'] = to_boolean(attrs['confidential'])
+        attrs.delete('confidential') if attrs['confidential'].nil?
+
         issue = ::Issues::UpdateService.new(user_project, current_user, attrs).execute(issue)
 
         if issue.valid?
-          # Find or create labels and attach to issue. Labels are valid because
-          # we already checked its name, so there can't be an error here
-          if params[:labels] && can?(current_user, :admin_issue, user_project)
-            issue.remove_labels
-            # Create and add labels to the new created issue
-            issue.add_labels_by_names(params[:labels].split(','))
-          end
-
           present issue, with: Entities::Issue, current_user: current_user
         else
           render_validation_error!(issue)
diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb
new file mode 100644
index 00000000000..07435d78468
--- /dev/null
+++ b/lib/api/merge_request_diffs.rb
@@ -0,0 +1,45 @@
+module API
+  # MergeRequestDiff API
+  class MergeRequestDiffs < Grape::API
+    before { authenticate! }
+
+    resource :projects do
+      desc 'Get a list of merge request diff versions' do
+        detail 'This feature was introduced in GitLab 8.12.'
+        success Entities::MergeRequestDiff
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+        requires :merge_request_id, type: Integer, desc: 'The ID of a merge request'
+      end
+
+      get ":id/merge_requests/:merge_request_id/versions" do
+        merge_request = user_project.merge_requests.
+          find(params[:merge_request_id])
+
+        authorize! :read_merge_request, merge_request
+        present merge_request.merge_request_diffs, with: Entities::MergeRequestDiff
+      end
+
+      desc 'Get a single merge request diff version' do
+        detail 'This feature was introduced in GitLab 8.12.'
+        success Entities::MergeRequestDiffFull
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+        requires :merge_request_id, type: Integer, desc: 'The ID of a merge request'
+        requires :version_id, type: Integer, desc: 'The ID of a merge request diff version'
+      end
+
+      get ":id/merge_requests/:merge_request_id/versions/:version_id" do
+        merge_request = user_project.merge_requests.
+          find(params[:merge_request_id])
+
+        authorize! :read_merge_request, merge_request
+        present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull
+      end
+    end
+  end
+end
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index 3f63cd678e8..14f5be3b5f6 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -46,6 +46,7 @@ module API
           :note_events,
           :build_events,
           :pipeline_events,
+          :wiki_page_events,
           :enable_ssl_verification
         ]
         @hook = user_project.hooks.new(attrs)
@@ -80,6 +81,7 @@ module API
           :note_events,
           :build_events,
           :pipeline_events,
+          :wiki_page_events,
           :enable_ssl_verification
         ]
 
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 60cfc103afd..a1fd598414a 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -51,7 +51,7 @@ module API
         @projects = current_user.viewable_starred_projects
         @projects = filter_projects(@projects)
         @projects = paginate @projects
-        present @projects, with: Entities::Project
+        present @projects, with: Entities::Project, user: current_user
       end
 
       # Get all projects for admin user
@@ -105,6 +105,7 @@ module API
       #   visibility_level (optional) - 0 by default
       #   import_url (optional)
       #   public_builds (optional)
+      #   lfs_enabled (optional)
       # Example Request
       #   POST /projects
       post do
@@ -123,7 +124,9 @@ module API
                                      :public,
                                      :visibility_level,
                                      :import_url,
-                                     :public_builds]
+                                     :public_builds,
+                                     :only_allow_merge_if_build_succeeds,
+                                     :lfs_enabled]
         attrs = map_public_to_visibility_level(attrs)
         @project = ::Projects::CreateService.new(current_user, attrs).execute
         if @project.saved?
@@ -155,6 +158,7 @@ module API
       #   visibility_level (optional)
       #   import_url (optional)
       #   public_builds (optional)
+      #   lfs_enabled (optional)
       # Example Request
       #   POST /projects/user/:user_id
       post "user/:user_id" do
@@ -172,7 +176,9 @@ module API
                                      :public,
                                      :visibility_level,
                                      :import_url,
-                                     :public_builds]
+                                     :public_builds,
+                                     :only_allow_merge_if_build_succeeds,
+                                     :lfs_enabled]
         attrs = map_public_to_visibility_level(attrs)
         @project = ::Projects::CreateService.new(user, attrs).execute
         if @project.saved?
@@ -218,6 +224,7 @@ module API
       #   public (optional) - if true same as setting visibility_level = 20
       #   visibility_level (optional) - visibility level of a project
       #   public_builds (optional)
+      #   lfs_enabled (optional)
       # Example Request
       #   PUT /projects/:id
       put ':id' do
@@ -234,7 +241,9 @@ module API
                                      :shared_runners_enabled,
                                      :public,
                                      :visibility_level,
-                                     :public_builds]
+                                     :public_builds,
+                                     :only_allow_merge_if_build_succeeds,
+                                     :lfs_enabled]
         attrs = map_public_to_visibility_level(attrs)
         authorize_admin_project
         authorize! :rename_project, user_project if attrs[:name].present?
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 8a376d3c2a3..c440305ff0f 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -327,7 +327,7 @@ module API
       # Example Request:
       #   GET /user
       get do
-        present @current_user, with: Entities::UserLogin
+        present @current_user, with: Entities::UserFull
       end
 
       # Get currently authenticated user's keys