Merge branch 'master' into faster-diffsfaster-diffs

author: Jacob Schatz <jschatz1@gmail.com> 2016-07-11 17:19:17 -0400
committer: Jacob Schatz <jschatz1@gmail.com> 2016-07-11 17:19:17 -0400
commit: 0452e0a57e24fdd65f559cc1a8629892714adc7a (patch)
tree: bf33ca472a3d434770783d80b5d10211383e706c /lib
parent: 7690513495d02d14af8a1a0cb3bf00f243ec0419 (diff)
parent: f76596380fb545c54b14c6bfc339a68a1dc162d3 (diff)
download: gitlab-ce-faster-diffs.tar.gz
13 files changed, 160 insertions, 23 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index db877d2eeb0..8edb80177da 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -58,6 +58,14 @@ module API
       expose :path, :path_with_namespace
     end
 
+    class SharedGroup < Grape::Entity
+      expose :group_id
+      expose :group_name do |group_link, options|
+        group_link.group.name
+      end
+      expose :group_access, as: :group_access_level
+    end
+
     class Project < Grape::Entity
       expose :id, :description, :default_branch, :tag_list
       expose :public?, as: :public
@@ -77,6 +85,9 @@ module API
       expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds
+      expose :shared_with_groups do |project, options|
+        SharedGroup.represent(project.project_group_links.all, options)
+      end
     end
 
     class ProjectMember < UserBasic
@@ -93,6 +104,7 @@ module API
 
     class GroupDetail < Group
       expose :projects, using: Entities::Project
+      expose :shared_projects, using: Entities::Project
     end
 
     class GroupMember < UserBasic
@@ -187,7 +199,6 @@ module API
       expose :author, :assignee, using: Entities::UserBasic
       expose :source_project_id, :target_project_id
       expose :label_names, as: :labels
-      expose :description
       expose :work_in_progress?, as: :work_in_progress
       expose :milestone, using: Entities::Milestone
       expose :merge_when_build_succeeds
diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb
new file mode 100644
index 00000000000..d2c4b1e4d76
--- /dev/null
+++ b/lib/banzai/filter/blockquote_fence_filter.rb
@@ -0,0 +1,71 @@
+module Banzai
+  module Filter
+    class BlockquoteFenceFilter < HTML::Pipeline::TextFilter
+      REGEX = %r{
+          (?<code>
+            # Code blocks:
+            # ```
+            # Anything, including `>>>` blocks which are ignored by this filter
+            # ```
+
+            ^```
+            .+?
+            \n```$
+          )
+        |
+          (?<html>
+            # HTML block:
+            # <tag>
+            # Anything, including `>>>` blocks which are ignored by this filter
+            # </tag>
+
+            ^<[^>]+?>\n
+            .+?
+            \n<\/[^>]+?>$
+          )
+        |
+          (?:
+            # Blockquote:
+            # >>>
+            # Anything, including code and HTML blocks
+            # >>>
+
+            ^>>>\n
+            (?<quote>
+              (?:
+                  # Any character that doesn't introduce a code or HTML block
+                  (?!
+                      ^```
+                    |
+                      ^<[^>]+?>\n
+                  )
+                  .
+                |
+                  # A code block
+                  \g<code>
+                |
+                  # An HTML block
+                  \g<html>
+              )+?
+            )
+            \n>>>$
+          )
+      }mx.freeze
+
+      def initialize(text, context = nil, result = nil)
+        super text, context, result
+        @text = @text.delete("\r")
+      end
+
+      def call
+        @text.gsub(REGEX) do
+          if $~[:quote]
+            $~[:quote].gsub(/^/, "> ").gsub(/^> $/, ">")
+          else
+            $~[0]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb
index 50dc978b452..6cf219661d3 100644
--- a/lib/banzai/pipeline/pre_process_pipeline.rb
+++ b/lib/banzai/pipeline/pre_process_pipeline.rb
@@ -3,7 +3,8 @@ module Banzai
     class PreProcessPipeline < BasePipeline
       def self.filters
         FilterArray[
-          Filter::YamlFrontMatterFilter
+          Filter::YamlFrontMatterFilter,
+          Filter::BlockquoteFenceFilter,
         ]
       end
 
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index 3d7b9c4a024..6cf218aaa0d 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -133,8 +133,9 @@ module Banzai
         return {} if nodes.empty?
 
         ids = unique_attribute_values(nodes, attribute)
+        rows = collection_objects_for_ids(collection, ids)
 
-        collection.where(id: ids).each_with_object({}) do |row, hash|
+        rows.each_with_object({}) do |row, hash|
           hash[row.id] = row
         end
       end
@@ -153,6 +154,31 @@ module Banzai
         values.to_a
       end
 
+      # Queries the collection for the objects with the given IDs.
+      #
+      # If the RequestStore module is enabled this method will only query any
+      # objects that have not yet been queried. For objects that have already
+      # been queried the object is returned from the cache.
+      def collection_objects_for_ids(collection, ids)
+        if RequestStore.active?
+          cache = collection_cache[collection_cache_key(collection)]
+          to_query = ids.map(&:to_i) - cache.keys
+
+          unless to_query.empty?
+            collection.where(id: to_query).each { |row| cache[row.id] = row }
+          end
+
+          cache.values
+        else
+          collection.where(id: ids)
+        end
+      end
+
+      # Returns the cache key to use for a collection.
+      def collection_cache_key(collection)
+        collection.respond_to?(:model) ? collection.model : collection
+      end
+
       # Processes the list of HTML documents and returns an Array containing all
       # the references.
       def process(documents)
@@ -189,7 +215,7 @@ module Banzai
       end
 
       def find_projects_for_hash_keys(hash)
-        Project.where(id: hash.keys)
+        collection_objects_for_ids(Project, hash.keys)
       end
 
       private
@@ -199,6 +225,12 @@ module Banzai
       def lazy(&block)
         Gitlab::Lazy.new(&block)
       end
+
+      def collection_cache
+        RequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key|
+          hash[key] = {}
+        end
+      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb
index a12b0d19560..863f5725d3b 100644
--- a/lib/banzai/reference_parser/user_parser.rb
+++ b/lib/banzai/reference_parser/user_parser.rb
@@ -73,7 +73,7 @@ module Banzai
       def find_users(ids)
         return [] if ids.empty?
 
-        User.where(id: ids).to_a
+        collection_objects_for_ids(User, ids)
       end
 
       def find_users_for_groups(ids)
@@ -85,7 +85,8 @@ module Banzai
       def find_users_for_projects(ids)
         return [] if ids.empty?
 
-        Project.where(id: ids).flat_map { |p| p.team.members.to_a }
+        collection_objects_for_ids(Project, ids).
+          flat_map { |p| p.team.members.to_a }
       end
     end
   end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 54b46e5d23f..ffc1814b29d 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -48,6 +48,7 @@ module Gitlab
         akismet_enabled: false,
         repository_checks_enabled: true,
         container_registry_token_expire_delay: 5,
+        user_default_external: false,
       )
     end
 
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index b0c50edba59..7e01f7b61fb 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -68,6 +68,10 @@ module Gitlab
         @lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a
       end
 
+      def collapsed_by_default?
+        diff.diff.bytesize > 10240 # 10 KB
+      end
+
       def highlighted_diff_lines
         @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight
       end
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index 57c41b41298..9b681e636c7 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -15,7 +15,7 @@ module Gitlab
       end
 
       def trigger(gl_id, oldrev, newrev, ref)
-        return true unless exists?
+        return [true, nil] unless exists?
 
         case name
         when "pre-receive", "post-receive"
@@ -70,13 +70,10 @@ module Gitlab
       end
 
       def call_update_hook(gl_id, oldrev, newrev, ref)
-        status = nil
-
         Dir.chdir(repo_path) do
-          status = system({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev)
+          stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev)
+          [status.success?, stderr.presence || stdout]
         end
-
-        [status, nil]
       end
 
       def retrieve_error_message(stderr, stdout)
diff --git a/lib/gitlab/graphs/commits.rb b/lib/gitlab/graphs/commits.rb
index 2122339d2db..3caf9036459 100644
--- a/lib/gitlab/graphs/commits.rb
+++ b/lib/gitlab/graphs/commits.rb
@@ -18,7 +18,7 @@ module Gitlab
       end
 
       def commit_per_day
-        @commit_per_day ||= (@commits.size.to_f / @duration).round(1)
+        @commit_per_day ||= @commits.size / (@duration + 1)
       end
 
       def collect_data
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 0ac6ff01e3b..025ecc12f9f 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -69,10 +69,19 @@ module Gitlab
       # Example:
       # +relation_key+ issues, loops through the list of *issues* and for each individual
       # issue, finds any subrelations such as notes, creates them and assign them back to the hash
+      #
+      # Recursively calls this method if the sub-relation is a hash containing more sub-relations
       def create_sub_relations(relation, tree_hash)
         relation_key = relation.keys.first.to_s
+        return if tree_hash[relation_key].blank?
+
         tree_hash[relation_key].each do |relation_item|
           relation.values.flatten.each do |sub_relation|
+            # We just use author to get the user ID, do not attempt to create an instance.
+            next if sub_relation == :author
+
+            create_sub_relations(sub_relation, relation_item) if sub_relation.is_a?(Hash)
+
             relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
             relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
           end
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 86ed18fb50d..19dad699edf 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -4,6 +4,8 @@ module Gitlab
       regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git'])
 
       content.gsub(regexp) { |url| new(url).masked_url }
+    rescue Addressable::URI::InvalidURIError
+      content.gsub(regexp, '')
     end
 
     def self.valid?(url)
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index bc0193a6c32..6aeb49c0219 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -63,6 +63,18 @@ module Gitlab
         ]
       end
 
+      def send_artifacts_entry(build, entry)
+        params = {
+          'Archive' => build.artifacts_file.path,
+          'Entry' => Base64.encode64(entry.path)
+        }
+
+        [
+          SEND_DATA_HEADER,
+          "artifacts-entry:#{encode(params)}"
+        ]
+      end
+
       protected
 
       def encode(hash)
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
index 8c309efc7b8..3358ed6773e 100644
--- a/lib/rouge/formatters/html_gitlab.rb
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -143,18 +143,14 @@ module Rouge
               '</span>'
             end
           end
-          lines.join("\n")
-        else
-          if @linenos == 'inline'
-            lines = lines.each_with_index.map do |line, index|
-              number = index + @linenostart
-              "<span class=\"linenos\">#{number}</span>#{line}"
-            end
-            lines.join("\n")
-          else
-            lines.join("\n")
+        elsif @linenos == 'inline'
+          lines = lines.each_with_index.map do |line, index|
+            number = index + @linenostart
+            "<span class=\"linenos\">#{number}</span>#{line}"
           end
         end
+
+        lines.join("\n")
       end
 
       def span(tok, val)
author	Jacob Schatz <jschatz1@gmail.com>	2016-07-11 17:19:17 -0400
committer	Jacob Schatz <jschatz1@gmail.com>	2016-07-11 17:19:17 -0400
commit	0452e0a57e24fdd65f559cc1a8629892714adc7a (patch)
tree	bf33ca472a3d434770783d80b5d10211383e706c /lib
parent	7690513495d02d14af8a1a0cb3bf00f243ec0419 (diff)
parent	f76596380fb545c54b14c6bfc339a68a1dc162d3 (diff)
download	gitlab-ce-faster-diffs.tar.gz