Merge branch 'master' into 22211-500-instead-of-404

36 files changed, 518 insertions, 208 deletions

diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
index d3db7740830..87915b19480 100644
--- a/lib/api/access_requests.rb
+++ b/lib/api/access_requests.rb
@@ -5,15 +5,14 @@ module API
     helpers ::API::Helpers::MembersHelpers
 
     %w[group project].each do |source_type|
+      params do
+        requires :id, type: String, desc: "The #{source_type} ID"
+      end
       resource source_type.pluralize do
-        # Get a list of group/project access requests viewable by the authenticated user.
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #
-        # Example Request:
-        #  GET /groups/:id/access_requests
-        #  GET /projects/:id/access_requests
+        desc "Gets a list of access requests for a #{source_type}." do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success Entities::AccessRequester
+        end
         get ":id/access_requests" do
           source = find_source(source_type, params[:id])
 
@@ -23,14 +22,10 @@ module API
           present access_requesters.map(&:user), with: Entities::AccessRequester, source: source
         end
 
-        # Request access to the group/project
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #
-        # Example Request:
-        #  POST /groups/:id/access_requests
-        #  POST /projects/:id/access_requests
+        desc "Requests access for the authenticated user to a #{source_type}." do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success Entities::AccessRequester
+        end
         post ":id/access_requests" do
           source = find_source(source_type, params[:id])
           access_requester = source.request_access(current_user)
@@ -42,37 +37,30 @@ module API
           end
         end
 
-        # Approve a group/project access request
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #   user_id (required) - The user ID of the access requester
-        #   access_level (optional) - Access level
-        #
-        # Example Request:
-        #   PUT /groups/:id/access_requests/:user_id/approve
-        #   PUT /projects/:id/access_requests/:user_id/approve
+        desc 'Approves an access request for the given user.' do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success Entities::Member
+        end
+        params do
+          requires :user_id, type: Integer, desc: 'The user ID of the access requester'
+          optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)'
+        end
         put ':id/access_requests/:user_id/approve' do
-          required_attributes! [:user_id]
           source = find_source(source_type, params[:id])
 
-          member = ::Members::ApproveAccessRequestService.new(source, current_user, params).execute
+          member = ::Members::ApproveAccessRequestService.new(source, current_user, declared(params)).execute
 
           status :created
           present member.user, with: Entities::Member, member: member
         end
 
-        # Deny a group/project access request
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #   user_id (required) - The user ID of the access requester
-        #
-        # Example Request:
-        #   DELETE /groups/:id/access_requests/:user_id
-        #   DELETE /projects/:id/access_requests/:user_id
+        desc 'Denies an access request for the given user.' do
+          detail 'This feature was introduced in GitLab 8.11.'
+        end
+        params do
+          requires :user_id, type: Integer, desc: 'The user ID of the access requester'
+        end
         delete ":id/access_requests/:user_id" do
-          required_attributes! [:user_id]
           source = find_source(source_type, params[:id])
 
           ::Members::DestroyService.new(source, current_user, params).
diff --git a/lib/api/api.rb b/lib/api/api.rb
index cb47ec8f33f..99722a0a65c 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -43,6 +43,7 @@ module API
     mount ::API::Groups
     mount ::API::Internal
     mount ::API::Issues
+    mount ::API::Boards
     mount ::API::Keys
     mount ::API::Labels
     mount ::API::LicenseTemplates
@@ -72,5 +73,9 @@ module API
     mount ::API::Triggers
     mount ::API::Users
     mount ::API::Variables
+
+    route :any, '*path' do
+      error!('404 Not Found', 404)
+    end
   end
 end
diff --git a/lib/api/boards.rb b/lib/api/boards.rb
new file mode 100644
index 00000000000..4d5d144a02e
--- /dev/null
+++ b/lib/api/boards.rb
@@ -0,0 +1,115 @@
+module API
+  # Boards API
+  class Boards < Grape::API
+    before { authenticate! }
+
+    resource :projects do
+      # Get the project board
+      get ':id/boards' do
+        authorize!(:read_board, user_project)
+        present [user_project.board], with: Entities::Board
+      end
+
+      segment ':id/boards/:board_id' do
+        helpers do
+          def project_board
+            board = user_project.board
+            if params[:board_id].to_i == board.id
+              board
+            else
+              not_found!('Board')
+            end
+          end
+
+          def board_lists
+            project_board.lists.destroyable
+          end
+        end
+
+        # Get the lists of a project board
+        # Does not include `backlog` and `done` lists
+        get '/lists' do
+          authorize!(:read_board, user_project)
+          present board_lists, with: Entities::List
+        end
+
+        # Get a list of a project board
+        get '/lists/:list_id' do
+          authorize!(:read_board, user_project)
+          present board_lists.find(params[:list_id]), with: Entities::List
+        end
+
+        # Create a new board list
+        #
+        # Parameters:
+        #   id (required)           - The ID of a project
+        #   label_id (required)     - The ID of an existing label
+        # Example Request:
+        #   POST /projects/:id/boards/:board_id/lists
+        post '/lists' do
+          required_attributes! [:label_id]
+
+          unless user_project.labels.exists?(params[:label_id])
+            render_api_error!({ error: "Label not found!" }, 400)
+          end
+
+          authorize!(:admin_list, user_project)
+
+          list = ::Boards::Lists::CreateService.new(user_project, current_user,
+              { label_id: params[:label_id] }).execute
+
+          if list.valid?
+            present list, with: Entities::List
+          else
+            render_validation_error!(list)
+          end
+        end
+
+        # Moves a board list to a new position
+        #
+        # Parameters:
+        #   id (required) - The ID of a project
+        #   board_id (required) - The ID of a board
+        #   position (required) - The position of the list
+        # Example Request:
+        #   PUT /projects/:id/boards/:board_id/lists/:list_id
+        put '/lists/:list_id' do
+          list = project_board.lists.movable.find(params[:list_id])
+
+          authorize!(:admin_list, user_project)
+
+          moved = ::Boards::Lists::MoveService.new(user_project, current_user,
+              { position: params[:position].to_i }).execute(list)
+
+          if moved
+            present list, with: Entities::List
+          else
+            render_api_error!({ error: "List could not be moved!" }, 400)
+          end
+        end
+
+        # Delete a board list
+        #
+        # Parameters:
+        #   id (required) - The ID of a project
+        #   board_id (required) - The ID of a board
+        #   list_id (required) - The ID of a board list
+        # Example Request:
+        #   DELETE /projects/:id/boards/:board_id/lists/:list_id
+        delete "/lists/:list_id" do
+          list = board_lists.find_by(id: params[:list_id])
+
+          authorize!(:admin_list, user_project)
+
+          if list
+            destroyed_list = ::Boards::Lists::DestroyService.new(
+              user_project, current_user).execute(list)
+            present destroyed_list, with: Entities::List
+          else
+            not_found!('List')
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index b4eaf1813d4..14ddc8c9a62 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -29,6 +29,42 @@ module API
         present commits, with: Entities::RepoCommit
       end
 
+      desc 'Commit multiple file changes as one commit' do
+        detail 'This feature was introduced in GitLab 8.13'
+      end
+
+      params do
+        requires :id, type: Integer, desc: 'The project ID'
+        requires :branch_name, type: String, desc: 'The name of branch'
+        requires :commit_message, type: String, desc: 'Commit message'
+        requires :actions, type: Array, desc: 'Actions to perform in commit'
+        optional :author_email, type: String, desc: 'Author email for commit'
+        optional :author_name, type: String, desc: 'Author name for commit'
+      end
+
+      post ":id/repository/commits" do
+        authorize! :push_code, user_project
+
+        attrs = declared(params)
+        attrs[:source_branch] = attrs[:branch_name]
+        attrs[:target_branch] = attrs[:branch_name]
+        attrs[:actions].map! do |action|
+          action[:action] = action[:action].to_sym
+          action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/')
+          action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/')
+          action
+        end
+
+        result = ::Files::MultiService.new(user_project, current_user, attrs).execute
+
+        if result[:status] == :success
+          commit_detail = user_project.repository.commits(result[:result], limit: 1).first
+          present commit_detail, with: Entities::RepoCommitDetail
+        else
+          render_api_error!(result[:message], 400)
+        end
+      end
+
       # Get a specific commit of a project
       #
       # Parameters:
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 04437322ec1..feaa0c213bf 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -432,8 +432,11 @@ module API
       end
     end
 
-    class Label < Grape::Entity
+    class LabelBasic < Grape::Entity
       expose :name, :color, :description
+    end
+
+    class Label < LabelBasic
       expose :open_issues_count, :closed_issues_count, :open_merge_requests_count
 
       expose :subscribed do |label, options|
@@ -441,6 +444,19 @@ module API
       end
     end
 
+    class List < Grape::Entity
+      expose :id
+      expose :label, using: Entities::LabelBasic
+      expose :position
+    end
+
+    class Board < Grape::Entity
+      expose :id
+      expose :lists, using: Entities::List do |board|
+        board.lists.destroyable
+      end
+    end
+
     class Compare < Grape::Entity
       expose :commit, using: Entities::RepoCommit do |compare, options|
         Commit.decorate(compare.commits, nil).last
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 953fa474e88..bfb89475025 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -6,6 +6,8 @@ module API
     resource :groups do
       # Get a groups list
       #
+      # Parameters:
+      #   skip_groups (optional) - Array of group ids to exclude from list
       # Example Request:
       #  GET /groups
       get do
@@ -16,6 +18,7 @@ module API
                   end
 
         @groups = @groups.search(params[:search]) if params[:search].present?
+        @groups = @groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
         @groups = paginate @groups
         present @groups, with: Entities::Group
       end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 714d4ea3dc6..67473f300c9 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -21,8 +21,11 @@ module API
     end
 
     # Check the Rails session for valid authentication details
+    #
+    # Until CSRF protection is added to the API, disallow this method for
+    # state-changing endpoints
     def find_user_from_warden
-      warden ? warden.authenticate : nil
+      warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD'])
     end
 
     def find_user_by_private_token
@@ -430,7 +433,7 @@ module API
     end
 
     def secret_token
-      File.read(Gitlab.config.gitlab_shell.secret_file).chomp
+      Gitlab::Shell.secret_token
     end
 
     def send_git_blob(repository, blob)
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 34df55fe192..b80818f0eb6 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -5,16 +5,16 @@ module API
     helpers ::API::Helpers::MembersHelpers
 
     %w[group project].each do |source_type|
+      params do
+        requires :id, type: String, desc: "The #{source_type} ID"
+      end
       resource source_type.pluralize do
-        # Get a list of group/project members viewable by the authenticated user.
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #   query         - Query string
-        #
-        # Example Request:
-        #   GET /groups/:id/members
-        #   GET /projects/:id/members
+        desc 'Gets a list of group or project members viewable by the authenticated user.' do
+          success Entities::Member
+        end
+        params do
+          optional :query, type: String, desc: 'A query string to search for members'
+        end
         get ":id/members" do
           source = find_source(source_type, params[:id])
 
@@ -25,15 +25,12 @@ module API
           present users, with: Entities::Member, source: source
         end
 
-        # Get a group/project member
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #   user_id (required) - The user ID of the member
-        #
-        # Example Request:
-        #   GET /groups/:id/members/:user_id
-        #   GET /projects/:id/members/:user_id
+        desc 'Gets a member of a group or project.' do
+          success Entities::Member
+        end
+        params do
+          requires :user_id, type: Integer, desc: 'The user ID of the member'
+        end
         get ":id/members/:user_id" do
           source = find_source(source_type, params[:id])
 
@@ -43,26 +40,25 @@ module API
           present member.user, with: Entities::Member, member: member
         end
 
-        # Add a new group/project member
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #   user_id (required) - The user ID of the new member
-        #   access_level (required) - A valid access level
-        #   expires_at (optional) - Date string in the format YEAR-MONTH-DAY
-        #
-        # Example Request:
-        #   POST /groups/:id/members
-        #   POST /projects/:id/members
+        desc 'Adds a member to a group or project.' do
+          success Entities::Member
+        end
+        params do
+          requires :user_id, type: Integer, desc: 'The user ID of the new member'
+          requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)'
+          optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
+        end
         post ":id/members" do
           source = find_source(source_type, params[:id])
           authorize_admin_source!(source_type, source)
-          required_attributes! [:user_id, :access_level]
 
           member = source.members.find_by(user_id: params[:user_id])
 
-          # This is to ensure back-compatibility but 409 behavior should be used
-          # for both project and group members in 9.0!
+          # We need this explicit check because `source.add_user` doesn't
+          # currently return the member created so it would return 201 even if
+          # the member already existed...
+          # The `source_type == 'group'` check is to ensure back-compatibility
+          # but 409 behavior should be used for both project and group members in 9.0!
           conflict!('Member already exists') if source_type == 'group' && member
 
           unless member
@@ -79,21 +75,17 @@ module API
           end
         end
 
-        # Update a group/project member
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #   user_id (required) - The user ID of the member
-        #   access_level (required) - A valid access level
-        #   expires_at (optional) - Date string in the format YEAR-MONTH-DAY
-        #
-        # Example Request:
-        #   PUT /groups/:id/members/:user_id
-        #   PUT /projects/:id/members/:user_id
+        desc 'Updates a member of a group or project.' do
+          success Entities::Member
+        end
+        params do
+          requires :user_id, type: Integer, desc: 'The user ID of the new member'
+          requires :access_level, type: Integer, desc: 'A valid access level'
+          optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
+        end
         put ":id/members/:user_id" do
           source = find_source(source_type, params[:id])
           authorize_admin_source!(source_type, source)
-          required_attributes! [:user_id, :access_level]
 
           member = source.members.find_by!(user_id: params[:user_id])
           attrs = attributes_for_keys [:access_level, :expires_at]
@@ -108,18 +100,12 @@ module API
           end
         end
 
-        # Remove a group/project member
-        #
-        # Parameters:
-        #   id (required) - The group/project ID
-        #   user_id (required) - The user ID of the member
-        #
-        # Example Request:
-        #   DELETE /groups/:id/members/:user_id
-        #   DELETE /projects/:id/members/:user_id
+        desc 'Removes a user from a group or project.'
+        params do
+          requires :user_id, type: Integer, desc: 'The user ID of the member'
+        end
         delete ":id/members/:user_id" do
           source = find_source(source_type, params[:id])
-          required_attributes! [:user_id]
 
           # This is to ensure back-compatibility but find_by! should be used
           # in that casse in 9.0!
@@ -134,7 +120,7 @@ module API
           if member.nil?
             { message: "Access revoked", id: params[:user_id].to_i }
           else
-            ::Members::DestroyService.new(source, current_user, params).execute
+            ::Members::DestroyService.new(source, current_user, declared(params)).execute
 
             present member.user, with: Entities::Member, member: member
           end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 680055c95eb..c24e8e8bd9b 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -22,14 +22,25 @@ module API
       # Example Request:
       #   GET /projects
       get do
-        @projects = current_user.authorized_projects
-        @projects = filter_projects(@projects)
-        @projects = paginate @projects
-        if params[:simple]
-          present @projects, with: Entities::BasicProjectDetails, user: current_user
-        else
-          present @projects, with: Entities::ProjectWithAccess, user: current_user
-        end
+        projects = current_user.authorized_projects
+        projects = filter_projects(projects)
+        projects = paginate projects
+        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
+
+        present projects, with: entity, user: current_user
+      end
+
+      # Get a list of visible projects for authenticated user
+      #
+      # Example Request:
+      #   GET /projects/visible
+      get '/visible' do
+        projects = ProjectsFinder.new.execute(current_user)
+        projects = filter_projects(projects)
+        projects = paginate projects
+        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
+
+        present projects, with: entity, user: current_user
       end
 
       # Get an owned projects list for authenticated user
@@ -37,10 +48,10 @@ module API
       # Example Request:
       #   GET /projects/owned
       get '/owned' do
-        @projects = current_user.owned_projects
-        @projects = filter_projects(@projects)
-        @projects = paginate @projects
-        present @projects, with: Entities::ProjectWithAccess, user: current_user
+        projects = current_user.owned_projects
+        projects = filter_projects(projects)
+        projects = paginate projects
+        present projects, with: Entities::ProjectWithAccess, user: current_user
       end
 
       # Gets starred project for the authenticated user
@@ -48,10 +59,10 @@ module API
       # Example Request:
       #   GET /projects/starred
       get '/starred' do
-        @projects = current_user.viewable_starred_projects
-        @projects = filter_projects(@projects)
-        @projects = paginate @projects
-        present @projects, with: Entities::Project, user: current_user
+        projects = current_user.viewable_starred_projects
+        projects = filter_projects(projects)
+        projects = paginate projects
+        present projects, with: Entities::Project, user: current_user
       end
 
       # Get all projects for admin user
@@ -60,10 +71,10 @@ module API
       #   GET /projects/all
       get '/all' do
         authenticated_as_admin!
-        @projects = Project.all
-        @projects = filter_projects(@projects)
-        @projects = paginate @projects
-        present @projects, with: Entities::ProjectWithAccess, user: current_user
+        projects = Project.all
+        projects = filter_projects(projects)
+        projects = paginate projects
+        present projects, with: Entities::ProjectWithAccess, user: current_user
       end
 
       # Get a single project
diff --git a/lib/banzai.rb b/lib/banzai.rb
index 9ebe379f454..35ca234c1ba 100644
--- a/lib/banzai.rb
+++ b/lib/banzai.rb
@@ -3,6 +3,10 @@ module Banzai
     Renderer.render(text, context)
   end
 
+  def self.render_field(object, field)
+    Renderer.render_field(object, field)
+  end
+
   def self.cache_collection_render(texts_and_contexts)
     Renderer.cache_collection_render(texts_and_contexts)
   end
diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb
new file mode 100644
index 00000000000..e008fd428b0
--- /dev/null
+++ b/lib/banzai/filter/html_entity_filter.rb
@@ -0,0 +1,12 @@
+require 'erb'
+
+module Banzai
+  module Filter
+    # Text filter that escapes these HTML entities: & " < >
+    class HtmlEntityFilter < HTML::Pipeline::TextFilter
+      def call
+        ERB::Util.html_escape(text)
+      end
+    end
+  end
+end
diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb
index bab6a9934d1..2b7c10f1a0e 100644
--- a/lib/banzai/note_renderer.rb
+++ b/lib/banzai/note_renderer.rb
@@ -3,7 +3,7 @@ module Banzai
     # Renders a collection of Note instances.
     #
     # notes - The notes to render.
-    # project - The project to use for rendering/redacting.
+    # project - The project to use for redacting.
     # user - The user viewing the notes.
     # path - The request path.
     # wiki - The project's wiki.
@@ -13,8 +13,7 @@ module Banzai
                                     user,
                                     requested_path: path,
                                     project_wiki: wiki,
-                                    ref: git_ref,
-                                    pipeline: :note)
+                                    ref: git_ref)
 
       renderer.render(notes, :note)
     end
diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb
index 9aef807c152..9f8eb0931b8 100644
--- a/lib/banzai/object_renderer.rb
+++ b/lib/banzai/object_renderer.rb
@@ -1,28 +1,32 @@
 module Banzai
-  # Class for rendering multiple objects (e.g. Note instances) in a single pass.
+  # Class for rendering multiple objects (e.g. Note instances) in a single pass,
+  # using +render_field+ to benefit from caching in the database. Rendering and
+  # redaction are both performed.
   #
-  # Rendered Markdown is stored in an attribute in every object based on the
-  # name of the attribute containing the Markdown. For example, when the
-  # attribute `note` is rendered the HTML is stored in `note_html`.
+  # The unredacted HTML is generated according to the usual +render_field+
+  # policy, so specify the pipeline and any other context options on the model.
+  #
+  # The *redacted* (i.e., suitable for use) HTML is placed in an attribute
+  # named "redacted_<foo>", where <foo> is the name of the cache field for the
+  # chosen attribute.
+  #
+  # As an example, rendering the attribute `note` would place the unredacted
+  # HTML into `note_html` and the redacted HTML into `redacted_note_html`.
   class ObjectRenderer
     attr_reader :project, :user
 
-    # Make sure to set the appropriate pipeline in the `raw_context` attribute
-    # (e.g. `:note` for Note instances).
-    #
-    # project - A Project to use for rendering and redacting Markdown.
+    # project - A Project to use for redacting Markdown.
     # user - The user viewing the Markdown/HTML documents, if any.
-    # context - A Hash containing extra attributes to use in the rendering
-    #           pipeline.
-    def initialize(project, user = nil, raw_context = {})
+    # context - A Hash containing extra attributes to use during redaction
+    def initialize(project, user = nil, redaction_context = {})
       @project = project
       @user = user
-      @raw_context = raw_context
+      @redaction_context = redaction_context
     end
 
     # Renders and redacts an Array of objects.
     #
-    # objects - The objects to render
+    # objects - The objects to render.
     # attribute - The attribute containing the raw Markdown to render.
     #
     # Returns the same input objects.
@@ -32,7 +36,7 @@ module Banzai
 
       objects.each_with_index do |object, index|
         redacted_data = redacted[index]
-        object.__send__("#{attribute}_html=", redacted_data[:document].to_html.html_safe)
+        object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe)
         object.user_visible_reference_count = redacted_data[:visible_reference_count]
       end
     end
@@ -53,12 +57,8 @@ module Banzai
 
     # Returns a Banzai context for the given object and attribute.
     def context_for(object, attribute)
-      context = base_context.merge(cache_key: [object, attribute])
-
-      if object.respond_to?(:author)
-        context[:author] = object.author
-      end
-
+      context = base_context.dup
+      context = context.merge(object.banzai_render_context(attribute))
       context
     end
 
@@ -66,21 +66,16 @@ module Banzai
     #
     # Returns an Array of `Nokogiri::HTML::Document`.
     def render_attributes(objects, attribute)
-      strings_and_contexts = objects.map do |object|
+      objects.map do |object|
+        string = Banzai.render_field(object, attribute)
         context = context_for(object, attribute)
 
-        string = object.__send__(attribute)
-
-        { text: string, context: context }
-      end
-
-      Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index|
-        Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context])
+        Banzai::Pipeline[:relative_link].to_document(string, context)
       end
     end
 
     def base_context
-      @base_context ||= @raw_context.merge(current_user: user, project: project)
+      @base_context ||= @redaction_context.merge(current_user: user, project: project)
     end
   end
 end
diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb
index ba2555df98d..1929099931b 100644
--- a/lib/banzai/pipeline/single_line_pipeline.rb
+++ b/lib/banzai/pipeline/single_line_pipeline.rb
@@ -3,6 +3,7 @@ module Banzai
     class SingleLinePipeline < GfmPipeline
       def self.filters
         @filters ||= FilterArray[
+          Filter::HtmlEntityFilter,
           Filter::SanitizationFilter,
 
           Filter::EmojiFilter,
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index a4ae27eefd8..6924a293da8 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -31,6 +31,34 @@ module Banzai
       end
     end
 
+    # Convert a Markdown-containing field on an object into an HTML-safe String
+    # of HTML. This method is analogous to calling render(object.field), but it
+    # can cache the rendered HTML in the object, rather than Redis.
+    #
+    # The context to use is learned from the passed-in object by calling
+    # #banzai_render_context(field), and cannot be changed. Use #render, passing
+    # it the field text, if a custom rendering is needed. The generated context
+    # is returned along with the HTML.
+    def render_field(object, field)
+      html_field = object.markdown_cache_field_for(field)
+
+      html = object.__send__(html_field)
+      return html if html.present?
+
+      html = cacheless_render_field(object, field)
+      object.update_column(html_field, html) unless object.new_record? || object.destroyed?
+
+      html
+    end
+
+    # Same as +render_field+, but without consulting or updating the cache field
+    def cacheless_render_field(object, field)
+      text = object.__send__(field)
+      context = object.banzai_render_context(field)
+
+      cacheless_render(text, context)
+    end
+
     # Perform multiple render from an Array of Markdown String into an
     # Array of HTML-safe String of HTML.
     #
diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb
new file mode 100644
index 00000000000..ca39b1961ae
--- /dev/null
+++ b/lib/constraints/group_url_constrainer.rb
@@ -0,0 +1,7 @@
+require 'constraints/namespace_url_constrainer'
+
+class GroupUrlConstrainer < NamespaceUrlConstrainer
+  def find_resource(id)
+    Group.find_by_path(id)
+  end
+end
diff --git a/lib/constraints/namespace_url_constrainer.rb b/lib/constraints/namespace_url_constrainer.rb
new file mode 100644
index 00000000000..23920193743
--- /dev/null
+++ b/lib/constraints/namespace_url_constrainer.rb
@@ -0,0 +1,13 @@
+class NamespaceUrlConstrainer
+  def matches?(request)
+    id = request.path.sub(/\A\/+/, '').split('/').first.sub(/.atom\z/, '')
+
+    if id =~ Gitlab::Regex.namespace_regex
+      find_resource(id)
+    end
+  end
+
+  def find_resource(id)
+    Namespace.find_by_path(id)
+  end
+end
diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb
new file mode 100644
index 00000000000..504a0f5d93e
--- /dev/null
+++ b/lib/constraints/user_url_constrainer.rb
@@ -0,0 +1,7 @@
+require 'constraints/namespace_url_constrainer'
+
+class UserUrlConstrainer < NamespaceUrlConstrainer
+  def find_resource(id)
+    User.find_by('lower(username) = ?', id.downcase)
+  end
+end
diff --git a/lib/event_filter.rb b/lib/event_filter.rb
index 668d2fa41b3..96e70e37e8f 100644
--- a/lib/event_filter.rb
+++ b/lib/event_filter.rb
@@ -2,8 +2,8 @@ class EventFilter
   attr_accessor :params
 
   class << self
-    def default_filter
-      %w{ push issues merge_requests team}
+    def all
+      'all'
     end
 
     def push
@@ -35,18 +35,21 @@ class EventFilter
     return events unless params.present?
 
     filter = params.dup
-
     actions = []
-    actions << Event::PUSHED if filter.include? 'push'
-    actions << Event::MERGED if filter.include? 'merged'
 
-    if filter.include? 'team'
-      actions << Event::JOINED
-      actions << Event::LEFT
+    case filter
+    when EventFilter.push
+      actions = [Event::PUSHED]
+    when EventFilter.merged
+      actions = [Event::MERGED]
+    when EventFilter.comments
+      actions = [Event::COMMENTED]
+    when EventFilter.team
+      actions = [Event::JOINED, Event::LEFT]
+    when EventFilter.all
+      actions = [Event::PUSHED, Event::MERGED, Event::COMMENTED, Event::JOINED, Event::LEFT]
     end
 
-    actions << Event::COMMENTED if filter.include? 'comments'
-
     events.where(action: actions)
   end
 
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 79eac66b364..d0060fbaca1 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -17,6 +17,18 @@ module Gitlab
     end
 
     class << self
+      def secret_token
+        @secret_token ||= begin
+          File.read(Gitlab.config.gitlab_shell.secret_file).chomp
+        end
+      end
+
+      def ensure_secret_token!
+        return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret'))
+
+        generate_and_link_secret_token
+      end
+
       def version_required
         @version_required ||= File.read(Rails.root.
                                         join('GITLAB_SHELL_VERSION')).strip
@@ -25,6 +37,25 @@ module Gitlab
       def strip_key(key)
         key.split(/ /)[0, 2].join(' ')
       end
+
+      private
+
+      # Create (if necessary) and link the secret token file
+      def generate_and_link_secret_token
+        secret_file = Gitlab.config.gitlab_shell.secret_file
+        shell_path = Gitlab.config.gitlab_shell.path
+
+        unless File.size?(secret_file)
+          # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
+          token = SecureRandom.hex(16)
+          File.write(secret_file, token)
+        end
+
+        link_path = File.join(shell_path, '.gitlab_shell_secret')
+        if File.exist?(shell_path) && !File.exist?(link_path)
+          FileUtils.symlink(secret_file, link_path)
+        end
+      end
     end
 
     # Init new repository
@@ -201,21 +232,6 @@ module Gitlab
       File.exist?(full_path(storage, dir_name))
     end
 
-    # Create (if necessary) and link the secret token file
-    def generate_and_link_secret_token
-      secret_file = Gitlab.config.gitlab_shell.secret_file
-      unless File.size?(secret_file)
-        # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
-        token = SecureRandom.hex(16)
-        File.write(secret_file, token)
-      end
-
-      link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret')
-      if File.exist?(gitlab_shell_path) && !File.exist?(link_path)
-        FileUtils.symlink(secret_file, link_path)
-      end
-    end
-
     protected
 
     def gitlab_shell_path
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index e33ac61f5ae..7f424b74efb 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -102,9 +102,19 @@ module Gitlab
       def request(method, *args, &block)
         sleep rate_limit_sleep_time if rate_limit_exceed?
 
-        data = api.send(method, *args, &block)
-        yield data
+        data = api.send(method, *args)
+        return data unless data.is_a?(Array)
 
+        if block_given?
+          yield data
+          each_response_page(&block)
+        else
+          each_response_page { |page| data.concat(page) }
+          data
+        end
+      end
+
+      def each_response_page
         last_response = api.last_response
 
         while last_response.rels[:next]
diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb
new file mode 100644
index 00000000000..b9e4042220a
--- /dev/null
+++ b/lib/gitlab/import_export/attribute_cleaner.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module ImportExport
+    class AttributeCleaner
+      ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES
+
+      def self.clean!(relation_hash:)
+        relation_hash.reject! do |key, _value|
+          key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index e522a0fc8f6..f00c7460e82 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module ImportExport
     module CommandLineUtil
+      DEFAULT_MODE = 0700
+
       def tar_czf(archive:, dir:)
         tar_with_options(archive: archive, dir: dir, options: 'czf')
       end
@@ -21,6 +23,11 @@ module Gitlab
         execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args)
       end
 
+      def mkdir_p(path)
+        FileUtils.mkdir_p(path, mode: DEFAULT_MODE)
+        FileUtils.chmod(DEFAULT_MODE, path)
+      end
+
       private
 
       def tar_with_options(archive:, dir:, options:)
@@ -45,7 +52,7 @@ module Gitlab
         # if we are copying files, create the destination folder
         destination_folder = File.file?(source) ? File.dirname(destination) : destination
 
-        FileUtils.mkdir_p(destination_folder)
+        mkdir_p(destination_folder)
         FileUtils.copy_entry(source, destination)
         true
       end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index eca6e5b6d51..113895ba22c 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -15,7 +15,7 @@ module Gitlab
       end
 
       def import
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
 
         wait_for_archived_file do
           decompress_archive
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 35ff134ea19..5a109f24f9f 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -110,9 +110,10 @@ module Gitlab
       def create_relation(relation, relation_hash_list)
         relation_array = [relation_hash_list].flatten.map do |relation_hash|
           Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
-                                                       relation_hash: relation_hash.merge('project_id' => restored_project.id),
+                                                       relation_hash: relation_hash,
                                                        members_mapper: members_mapper,
-                                                       user: @user)
+                                                       user: @user,
+                                                       project_id: restored_project.id)
         end
 
         relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index 9153088e966..2fbf437ec26 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module ImportExport
     class ProjectTreeSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
       attr_reader :full_path
 
       def initialize(project:, shared:)
@@ -10,7 +12,7 @@ module Gitlab
       end
 
       def save
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
 
         File.write(full_path, project_json_tree)
         true
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 354ccd64696..9300f789e1b 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -13,6 +13,8 @@ module Gitlab
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
 
+      PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze
+
       BUILD_MODELS = %w[Ci::Build commit_status].freeze
 
       IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
@@ -25,9 +27,9 @@ module Gitlab
         new(*args).create
       end
 
-      def initialize(relation_sym:, relation_hash:, members_mapper:, user:)
+      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:)
         @relation_name = OVERRIDES[relation_sym] || relation_sym
-        @relation_hash = relation_hash.except('id', 'noteable_id')
+        @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id)
         @members_mapper = members_mapper
         @user = user
         @imported_object_retries = 0
@@ -153,7 +155,11 @@ module Gitlab
       end
 
       def parsed_relation_hash
-        @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
+        @parsed_relation_hash ||= begin
+          Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash)
+
+          @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
+        end
       end
 
       def set_st_diffs
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index d1e33ea8678..48a9a6fa5e2 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -12,7 +12,7 @@ module Gitlab
       def restore
         return true unless File.exist?(@path_to_bundle)
 
-        FileUtils.mkdir_p(path_to_repo)
+        mkdir_p(path_to_repo)
 
         git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks
       rescue => e
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index 331e14021e6..a7028a32570 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -20,7 +20,7 @@ module Gitlab
       private
 
       def bundle_to_disk
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
         git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
       rescue => e
         @shared.error(e)
diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb
index 9b642d740b7..7cf88298642 100644
--- a/lib/gitlab/import_export/version_saver.rb
+++ b/lib/gitlab/import_export/version_saver.rb
@@ -1,12 +1,14 @@
 module Gitlab
   module ImportExport
     class VersionSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
       def initialize(shared:)
         @shared = shared
       end
 
       def save
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
 
         File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
       rescue => e
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 6107420e4dd..1e6722a7bba 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -9,7 +9,7 @@ module Gitlab
       end
 
       def bundle_to_disk(full_path)
-        FileUtils.mkdir_p(@shared.export_path)
+        mkdir_p(@shared.export_path)
         git_bundle(repo_path: path_to_repo, bundle_path: full_path)
       rescue => e
         @shared.error(e)
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index 3faab937726..c649da8c426 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -24,10 +24,20 @@ module Gitlab
       end
 
       def with
-        @pool ||= ConnectionPool.new { ::Redis.new(params) }
+        @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
         @pool.with { |redis| yield redis }
       end
 
+      def pool_size
+        if Sidekiq.server?
+          # the pool will be used in a multi-threaded context
+          Sidekiq.options[:concurrency] + 5
+        else
+          # probably this is a Unicorn process, so single threaded
+          5
+        end
+      end
+
       def _raw_config
         return @_raw_config if defined?(@_raw_config)
 
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 776bbcbb5d0..0d30e1bb92e 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Regex
     extend self
 
-    NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])'.freeze
+    NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])(?<!\.git|\.atom)'.freeze
 
     def namespace_regex
       @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze
@@ -10,7 +10,7 @@ module Gitlab
 
     def namespace_regex_message
       "can contain only letters, digits, '_', '-' and '.'. " \
-      "Cannot start with '-' or end in '.'." \
+      "Cannot start with '-' or end in '.', '.git' or '.atom'." \
     end
 
     def namespace_name_regex
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 5d33f98e89e..594439a5d4b 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -111,7 +111,7 @@ module Gitlab
       def write_secret
         bytes = SecureRandom.random_bytes(SECRET_LENGTH)
         File.open(secret_path, 'w:BINARY', 0600) do |f|
-          f.chmod(0600)
+          f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op.
           f.write(Base64.strict_encode64(bytes))
         end
       end
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index 2214f855200..a95a3455a4a 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -1,22 +1,33 @@
 namespace :cache do
-  CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
-  REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan
+  namespace :clear do
+    REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
+    REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan
 
-  desc "GitLab | Clear redis cache"
-  task :clear => :environment do
-    Gitlab::Redis.with do |redis|
-      cursor = REDIS_SCAN_START_STOP
-      loop do
-        cursor, keys = redis.scan(
-          cursor,
-          match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", 
-          count: CLEAR_BATCH_SIZE
-        )
-  
-        redis.del(*keys) if keys.any?
-  
-        break if cursor == REDIS_SCAN_START_STOP
+    desc "GitLab | Clear redis cache"
+    task redis: :environment do
+      Gitlab::Redis.with do |redis|
+        cursor = REDIS_SCAN_START_STOP
+        loop do
+          cursor, keys = redis.scan(
+            cursor,
+            match: "#{Gitlab::Redis::CACHE_NAMESPACE}*",
+            count: REDIS_CLEAR_BATCH_SIZE
+          )
+
+          redis.del(*keys) if keys.any?
+
+          break if cursor == REDIS_SCAN_START_STOP
+        end
       end
     end
+
+    desc "GitLab | Clear database cache (in the background)"
+    task db: :environment do
+      ClearDatabaseCacheWorker.perform_async
+    end
+
+    task all: [:db, :redis]
   end
+
+  task clear: 'cache:clear:all'
 end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index bb7eb852f1b..210899882b4 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -78,7 +78,7 @@ namespace :gitlab do
         f.puts "PATH=#{ENV['PATH']}"
       end
 
-      Gitlab::Shell.new.generate_and_link_secret_token
+      Gitlab::Shell.ensure_secret_token!
     end
 
     desc "GitLab | Setup gitlab-shell"