From 5a5069969ce8e9184e36abbb7623bf474d5869e9 Mon Sep 17 00:00:00 2001
From: Jonathan Schoeffling <jonathan.schoeffling@lmco.com>
Date: Sun, 14 Jun 2015 18:04:20 -0400
Subject: Add support for searching commit log messages

Include the log messages of recent commits in project-level search
results, providing functionality similar to 'git log --grep'.

Update repository model rspec tests to validate the output of
Repository#commits_with_log_matching.
---
 lib/gitlab/project_search_results.rb | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 0a2be605af9..3bf98699bcb 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -20,6 +20,8 @@ module Gitlab
         Kaminari.paginate_array(blobs).page(page).per(per_page)
       when 'wiki_blobs'
         Kaminari.paginate_array(wiki_blobs).page(page).per(per_page)
+      when 'commits'
+        Kaminari.paginate_array(commits).page(page).per(per_page)
       else
         super
       end
@@ -27,7 +29,7 @@ module Gitlab
 
     def total_count
       @total_count ||= issues_count + merge_requests_count + blobs_count +
-                       notes_count + wiki_blobs_count
+                       notes_count + wiki_blobs_count + commits_count
     end
 
     def blobs_count
@@ -42,6 +44,10 @@ module Gitlab
       @wiki_blobs_count ||= wiki_blobs.count
     end
 
+    def commits_count
+      @commits_count ||= commits.count
+    end
+
     private
 
     def blobs
@@ -70,6 +76,10 @@ module Gitlab
       Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC')
     end
 
+    def commits
+      project.repository.commits_with_log_matching(query)
+    end
+
     def limit_project_ids
       [project.id]
     end
-- 
cgit v1.2.1


From afdb53baecb2ef6d57f8ed957213260825c67805 Mon Sep 17 00:00:00 2001
From: Mike Chmielewski <code@mikec.123mail.org>
Date: Wed, 21 Oct 2015 01:04:19 +0000
Subject: Added @commits to list of tags.

---
 lib/tasks/spinach.rake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
index c8881be0954..d5a96fd38f4 100644
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -5,7 +5,7 @@ namespace :spinach do
   task :project do
     cmds = [
       %W(rake gitlab:setup),
-      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
+      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits),
     ]
     run_commands(cmds)
   end
@@ -14,7 +14,7 @@ namespace :spinach do
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
+      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits),
     ]
     run_commands(cmds)
   end
@@ -33,4 +33,4 @@ def run_commands(cmds)
   cmds.each do |cmd|
     system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
   end
-end
+end
\ No newline at end of file
-- 
cgit v1.2.1


From b1f4aaa5e753e6e7cdefd84226839123df59b382 Mon Sep 17 00:00:00 2001
From: Michael Chmielewski <code@mikec.123mail.org>
Date: Tue, 27 Oct 2015 21:16:56 -0400
Subject: Trying to incorporate suggestions from comments on Merge Request 1661

---
 lib/gitlab/project_search_results.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 3bf98699bcb..54389f7d662 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -77,7 +77,7 @@ module Gitlab
     end
 
     def commits
-      project.repository.commits_with_log_matching(query)
+      project.repository.find_commits_with_matching_log(query)
     end
 
     def limit_project_ids
-- 
cgit v1.2.1


From c03da1caade637298ca96e59cea990b0827539c9 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 2 Nov 2015 14:44:06 +0100
Subject: Extend yml syntax for only and except to support specifying
 repository path

---
 lib/ci/gitlab_ci_yaml_processor.rb | 64 ++++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 30 deletions(-)

(limited to 'lib')

diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index efcd2faffc7..0f57a4f53ab 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -7,10 +7,11 @@ module Ci
     ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
     ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
 
-    attr_reader :before_script, :image, :services, :variables
+    attr_reader :before_script, :image, :services, :variables, :path
 
-    def initialize(config)
+    def initialize(config, path = nil)
       @config = YAML.load(config)
+      @path = path
 
       unless @config.is_a? Hash
         raise ValidationError, "YAML should be a hash"
@@ -63,26 +64,6 @@ module Ci
       end
     end
 
-    def process?(only_params, except_params, ref, tag)
-      return true if only_params.nil? && except_params.nil?
-
-      if only_params
-        return true if tag && only_params.include?("tags")
-        return true if !tag && only_params.include?("branches")
-        
-        only_params.find do |pattern|
-          match_ref?(pattern, ref)
-        end
-      else
-        return false if tag && except_params.include?("tags")
-        return false if !tag && except_params.include?("branches")
-
-        except_params.each do |pattern|
-          return false if match_ref?(pattern, ref)
-        end
-      end
-    end
-
     def build_job(name, job)
       {
         stage_idx: stages.index(job[:stage]),
@@ -101,14 +82,6 @@ module Ci
       }
     end
 
-    def match_ref?(pattern, ref)
-      if pattern.first == "/" && pattern.last == "/"
-        Regexp.new(pattern[1...-1]) =~ ref
-      else
-        pattern == ref
-      end
-    end
-
     def normalize_script(script)
       if script.is_a? Array
         script.join("\n")
@@ -208,5 +181,36 @@ module Ci
     def validate_string(value)
       value.is_a?(String) || value.is_a?(Symbol)
     end
+
+    def process?(only_params, except_params, ref, tag)
+      if only_params.present?
+        return false unless matching?(only_params, ref, tag)
+      end
+
+      if except_params.present?
+        return false if matching?(except_params, ref, tag)
+      end
+
+      true
+    end
+
+    def matching?(patterns, ref, tag)
+      patterns.any? do |pattern|
+        match_ref?(pattern, ref, tag)
+      end
+    end
+
+    def match_ref?(pattern, ref, tag)
+      pattern, path = pattern.split('@', 2)
+      return false if path && path != self.path
+      return true if tag && pattern == 'tags'
+      return true if !tag && pattern == 'branches'
+
+      if pattern.first == "/" && pattern.last == "/"
+        Regexp.new(pattern[1...-1]) =~ ref
+      else
+        pattern == ref
+      end
+    end
   end
 end
-- 
cgit v1.2.1


From e032a7288f4103a6d5980da6af4040bc9271bd99 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Mon, 2 Nov 2015 16:02:47 +0100
Subject: Spread out runner contacted_at updates

---
 lib/ci/api/helpers.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index e602cda81d6..7e4986b6af3 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -16,7 +16,9 @@ module Ci
       end
 
       def update_runner_last_contact
-        if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY
+        # Use a random threshold to prevent beating DB updates
+        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
+        if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age
           current_runner.update_attributes(contacted_at: Time.now)
         end
       end
-- 
cgit v1.2.1


From 810c91fe35db6a83c9c517e03d07dc1795922646 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Mon, 2 Nov 2015 16:39:24 +0100
Subject: Refactor search by commits message

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/project_search_results.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 54389f7d662..c873ef3771e 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -77,7 +77,7 @@ module Gitlab
     end
 
     def commits
-      project.repository.find_commits_with_matching_log(query)
+      project.repository.find_commits_by_message(query)
     end
 
     def limit_project_ids
-- 
cgit v1.2.1


From 60c5b52ee906639e8bc0408a4a63d50862580fc8 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Mon, 2 Nov 2015 15:04:43 -0500
Subject: Explicitly require backup/files

---
 lib/backup/builds.rb  | 2 ++
 lib/backup/uploads.rb | 2 ++
 2 files changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
index 800f30c2144..635967f4bd4 100644
--- a/lib/backup/builds.rb
+++ b/lib/backup/builds.rb
@@ -1,3 +1,5 @@
+require 'backup/files'
+
 module Backup
   class Builds < Files
     def initialize
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index 0a0ec564ba4..9261f77f3c9 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -1,3 +1,5 @@
+require 'backup/files'
+
 module Backup
   class Uploads < Files
 
-- 
cgit v1.2.1


From 28f6fba97cb9753c43d109ef4f43439413d4eb69 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Tue, 3 Nov 2015 10:16:09 +0100
Subject: Fix commits search for empty repository

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/project_search_results.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index c873ef3771e..70de6a74e76 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -77,7 +77,11 @@ module Gitlab
     end
 
     def commits
-      project.repository.find_commits_by_message(query)
+      if project.empty_repo? || query.blank?
+        []
+      else
+        project.repository.find_commits_by_message(query).compact
+      end
     end
 
     def limit_project_ids
-- 
cgit v1.2.1


From 9479496f7584336788fc61121c096dfbcb3f6e4a Mon Sep 17 00:00:00 2001
From: Valery Sizov <valery@gitlab.com>
Date: Mon, 2 Nov 2015 16:56:44 +0200
Subject: Add added, modified and removed properties to commit object in
 webhook

---
 lib/gitlab/push_data_builder.rb | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb
index d010ade704e..fa068d50763 100644
--- a/lib/gitlab/push_data_builder.rb
+++ b/lib/gitlab/push_data_builder.rb
@@ -18,7 +18,10 @@ module Gitlab
       #     homepage: String,
       #   },
       #   commits: Array,
-      #   total_commits_count: Fixnum
+      #   total_commits_count: Fixnum,
+      #   added: ["CHANGELOG"],
+      #   modified: [],
+      #   removed: ["tmp/file.txt"]
       # }
       #
       def build(project, user, oldrev, newrev, ref, commits = [], message = nil)
@@ -33,6 +36,8 @@ module Gitlab
         commit_attrs = commits_limited.map(&:hook_attrs)
 
         type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push"
+
+        repo_changes = repo_changes(project, newrev, oldrev)
         # Hash to be passed as post_receive_data
         data = {
           object_kind: type,
@@ -55,7 +60,10 @@ module Gitlab
             visibility_level: project.visibility_level
           },
           commits: commit_attrs,
-          total_commits_count: commits_count
+          total_commits_count: commits_count,
+          added: repo_changes[:added],
+          modified: repo_changes[:modified],
+          removed: repo_changes[:removed]
         }
 
         data
@@ -86,6 +94,27 @@ module Gitlab
           newrev
         end
       end
+
+      def repo_changes(project, newrev, oldrev)
+        changes = { added: [], modified: [], removed: [] }
+        compare_result = CompareService.new.
+          execute(project, newrev, project, oldrev)
+
+        if compare_result
+          compare_result.diffs.each do |diff|
+            case true
+            when diff.deleted_file
+              changes[:removed] << diff.old_path
+            when diff.renamed_file, diff.new_file
+              changes[:added] << diff.new_path
+            else
+              changes[:modified] << diff.new_path
+            end
+          end
+        end
+
+        changes
+      end
     end
   end
 end
-- 
cgit v1.2.1


From 98dcad2762930b0dc967ff8e67f9f795228d2342 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Tue, 3 Nov 2015 17:58:26 +0100
Subject: Use proper labels for OAuth providers

---
 lib/gitlab/o_auth/provider.rb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb
index 90c3fe8da33..9ad7a38d505 100644
--- a/lib/gitlab/o_auth/provider.rb
+++ b/lib/gitlab/o_auth/provider.rb
@@ -1,6 +1,12 @@
 module Gitlab
   module OAuth
     class Provider
+      LABELS = {
+        "github"         => "GitHub",
+        "gitlab"         => "GitLab.com",
+        "google_oauth2"  => "Google"
+      }.freeze
+
       def self.providers
         Devise.omniauth_providers
       end
@@ -23,8 +29,9 @@ module Gitlab
       end
 
       def self.label_for(name)
+        name = name.to_s
         config = config_for(name)
-        (config && config['label']) || name.to_s.titleize
+        (config && config['label']) || LABELS[name] || name.titleize
       end
     end
   end
-- 
cgit v1.2.1


From d09d62b6b875102b7a334fcf9e689537e1f25013 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Tue, 3 Nov 2015 17:10:38 -0500
Subject: Replace all usages of `git` command with configurable binary path

Closes #3311
---
 lib/backup/repository.rb        |  6 +++---
 lib/gitlab/force_push_check.rb  |  2 +-
 lib/gitlab/git_ref_validator.rb |  2 +-
 lib/gitlab/upgrader.rb          |  8 ++++----
 lib/tasks/gitlab/check.rake     |  2 +-
 lib/tasks/gitlab/shell.rake     | 10 +++++-----
 6 files changed, 15 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 4d70f7883dd..a82a7e1f7bf 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -35,7 +35,7 @@ module Backup
           if wiki.repository.empty?
             $progress.puts " [SKIPPED]".cyan
           else
-            cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
+            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
             output, status = Gitlab::Popen.popen(cmd)
             if status.zero?
               $progress.puts " [DONE]".green
@@ -67,7 +67,7 @@ module Backup
           FileUtils.mkdir_p(path_to_repo(project))
           cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)})
         else
-          cmd = %W(git init --bare #{path_to_repo(project)})
+          cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)})
         end
 
         if system(*cmd, silent)
@@ -87,7 +87,7 @@ module Backup
           # that was initialized with ProjectWiki.new() and then
           # try to restore with 'git clone --bare'.
           FileUtils.rm_rf(path_to_repo(wiki))
-          cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
+          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
 
           if system(*cmd, silent)
             $progress.puts " [DONE]".green
diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb
index fdb6a35c78d..93c6a5bb7f5 100644
--- a/lib/gitlab/force_push_check.rb
+++ b/lib/gitlab/force_push_check.rb
@@ -7,7 +7,7 @@ module Gitlab
       if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
         false
       else
-        missed_refs, _ = Gitlab::Popen.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev}))
+        missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev}))
         missed_refs.split("\n").size > 0
       end
     end
diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb
index 39d17def930..4d83d8e72a8 100644
--- a/lib/gitlab/git_ref_validator.rb
+++ b/lib/gitlab/git_ref_validator.rb
@@ -6,7 +6,7 @@ module Gitlab
     # Returns true for a valid reference name, false otherwise
     def validate(ref_name)
       Gitlab::Utils.system_silent(
-        %W(git check-ref-format refs/#{ref_name}))
+        %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name}))
     end
   end
 end
diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb
index cf040971c6e..f3567f3ef85 100644
--- a/lib/gitlab/upgrader.rb
+++ b/lib/gitlab/upgrader.rb
@@ -50,15 +50,15 @@ module Gitlab
     end
 
     def fetch_git_tags
-      remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git))
+      remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git))
       remote_tags.split("\n").grep(/tags\/v#{current_version.major}/)
     end
 
     def update_commands
       {
-        "Stash changed files" => %W(git stash),
-        "Get latest code" => %W(git fetch),
-        "Switch to new version" => %W(git checkout v#{latest_version}),
+        "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash),
+        "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch),
+        "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}),
         "Install gems" => %W(bundle),
         "Migrate DB" => %W(bundle exec rake db:migrate),
         "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile),
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 2e73f792a9d..a25fac62cfc 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -824,7 +824,7 @@ namespace :gitlab do
         repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
         repo_dirs.each do |dir|
           puts "\nChecking repo at #{dir}"
-          system(*%w(git fsck), chdir: dir)
+          system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir)
         end
       end
     end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 3c0cc763d17..dd61632e557 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -17,7 +17,7 @@ namespace :gitlab do
 
       # Clone if needed
       unless File.directory?(target_dir)
-        system(*%W(git clone -- #{args.repo} #{target_dir}))
+        system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir}))
       end
 
       # Make sure we're on the right tag
@@ -27,7 +27,7 @@ namespace :gitlab do
         reseted = reset_to_commit(args)
 
         unless reseted
-          system(*%W(git fetch origin))
+          system(*%W(#{Gitlab.config.git.bin_path} fetch origin))
           reset_to_commit(args)
         end
 
@@ -128,14 +128,14 @@ namespace :gitlab do
   end
 
   def reset_to_commit(args)
-    tag, status = Gitlab::Popen.popen(%W(git describe -- #{args.tag}))
+    tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag}))
 
     unless status.zero?
-      tag, status = Gitlab::Popen.popen(%W(git describe -- origin/#{args.tag}))
+      tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag}))
     end
 
     tag = tag.strip
-    system(*%W(git reset --hard #{tag}))
+    system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
   end
 end
 
-- 
cgit v1.2.1


From 91cbf9db0c5c95381e7d422fd684e623d58fadab Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Mon, 26 Oct 2015 07:26:03 +0100
Subject: Add allow_failure field to commit status API

Closes #3196
---
 lib/api/entities.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 883a5e14b17..20cadae2291 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -231,7 +231,7 @@ module API
 
     class CommitStatus < Grape::Entity
       expose :id, :sha, :ref, :status, :name, :target_url, :description,
-             :created_at, :started_at, :finished_at
+             :created_at, :started_at, :finished_at, :allow_failure
       expose :author, using: Entities::UserBasic
     end
 
-- 
cgit v1.2.1


From b18671a1b2c565a87663544441000063f6b83c8e Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 3 Nov 2015 14:45:41 +0100
Subject: Enable shared runners for all new projects

---
 lib/gitlab/current_settings.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 0ea1b6a2f6f..cd84afa31d5 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -23,7 +23,8 @@ module Gitlab
         restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],
         max_attachment_size: Settings.gitlab['max_attachment_size'],
         session_expire_delay: Settings.gitlab['session_expire_delay'],
-        import_sources: Settings.gitlab['import_sources']
+        import_sources: Settings.gitlab['import_sources'],
+        shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
       )
     end
 
-- 
cgit v1.2.1


From dec3e4ce64df5f71a7cba7734cada1baa79242cd Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Wed, 4 Nov 2015 19:13:19 +0100
Subject: Added Sherlock, a custom profiling tool for GitLab

Sherlock will be a new GitLab specific tool for measuring the
performance of Rails requests (and SideKiq jobs at some point). Some of
the things that are currently tracked:

* SQL queries along with their timings, backtraces and query plans
  (using "EXPLAIN ANALYZE" for PostgreSQL and regular "EXPLAIN" for
  MySQL)
* Timings of application files (including views) on a per line basis
* Some meta data such as the request method, path, total duration, etc

More tracking (e.g. Rugged or gitlab-shell timings) might be added in
the future.

Sherlock will replace any existing tools we have used so far (e.g.
active_record_query_trace and rack-mini-profiler), hence the
corresponding Gems have been removed from the Gemfile.

Sherlock can be enabled by starting Rails as following:

    ENABLE_SHERLOCK=1 bundle exec rails s

Recorded transactions can be found at `/sherlock/transactions`.
---
 lib/gitlab/sherlock.rb               | 20 ++++++++
 lib/gitlab/sherlock/collection.rb    | 42 ++++++++++++++++
 lib/gitlab/sherlock/file_sample.rb   | 27 ++++++++++
 lib/gitlab/sherlock/line_profiler.rb | 60 ++++++++++++++++++++++
 lib/gitlab/sherlock/line_sample.rb   | 20 ++++++++
 lib/gitlab/sherlock/location.rb      | 22 +++++++++
 lib/gitlab/sherlock/middleware.rb    | 36 ++++++++++++++
 lib/gitlab/sherlock/query.rb         | 96 ++++++++++++++++++++++++++++++++++++
 lib/gitlab/sherlock/transaction.rb   | 85 +++++++++++++++++++++++++++++++
 9 files changed, 408 insertions(+)
 create mode 100644 lib/gitlab/sherlock.rb
 create mode 100644 lib/gitlab/sherlock/collection.rb
 create mode 100644 lib/gitlab/sherlock/file_sample.rb
 create mode 100644 lib/gitlab/sherlock/line_profiler.rb
 create mode 100644 lib/gitlab/sherlock/line_sample.rb
 create mode 100644 lib/gitlab/sherlock/location.rb
 create mode 100644 lib/gitlab/sherlock/middleware.rb
 create mode 100644 lib/gitlab/sherlock/query.rb
 create mode 100644 lib/gitlab/sherlock/transaction.rb

(limited to 'lib')

diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb
new file mode 100644
index 00000000000..c4b35b24ceb
--- /dev/null
+++ b/lib/gitlab/sherlock.rb
@@ -0,0 +1,20 @@
+require 'securerandom'
+require 'rblineprof' if RUBY_ENGINE == 'ruby'
+
+module Gitlab
+  module Sherlock
+    @collection = Collection.new
+
+    class << self
+      attr_reader :collection
+    end
+
+    def self.enabled?
+      Rails.env.development? && !!ENV['ENABLE_SHERLOCK']
+    end
+
+    def self.enable_line_profiler?
+      RUBY_ENGINE == 'ruby'
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb
new file mode 100644
index 00000000000..accdc6469bc
--- /dev/null
+++ b/lib/gitlab/sherlock/collection.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module Sherlock
+    class Collection
+      include Enumerable
+
+      def initialize
+        @transactions = []
+        @mutex = Mutex.new
+      end
+
+      def add(transaction)
+        synchronize { @transactions << transaction }
+      end
+
+      alias_method :<<, :add
+
+      def each(&block)
+        synchronize { @transactions.each(&block) }
+      end
+
+      def clear
+        synchronize { @transactions.clear }
+      end
+
+      def empty?
+        synchronize { @transactions.empty? }
+      end
+
+      def find_transaction(id)
+        find { |trans| trans.id == id }
+      end
+
+      def newest_first
+        sort { |a, b| b.finished_at <=> a.finished_at }
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb
new file mode 100644
index 00000000000..7a220de9abc
--- /dev/null
+++ b/lib/gitlab/sherlock/file_sample.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Sherlock
+    class FileSample
+      attr_reader :id, :file, :line_samples, :events, :duration
+
+      def initialize(file, line_samples, duration, events)
+        @id = SecureRandom.uuid
+        @file = file
+        @line_samples = line_samples
+        @duration = duration
+        @events = events
+      end
+
+      def relative_path
+        @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '')
+      end
+
+      def to_param
+        @id
+      end
+
+      def source
+        @source ||= File.read(@file)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
new file mode 100644
index 00000000000..a191b1e646d
--- /dev/null
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -0,0 +1,60 @@
+module Gitlab
+  module Sherlock
+    class LineProfiler
+      # The minimum amount of time that has to be spent in a file for it to be
+      # included in a list of samples.
+      MINIMUM_DURATION = 10.0
+
+      def profile(&block)
+        if RUBY_ENGINE == 'ruby'
+          profile_mri(&block)
+        else
+          raise NotImplementedError,
+            'Line profiling is not supported on this platform'
+        end
+      end
+
+      def profile_mri
+        retval  = nil
+        samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield }
+
+        file_samples = aggregate_rblineprof(samples)
+
+        [retval, file_samples]
+      end
+
+      # Returns an Array of file samples based on the output of rblineprof.
+      def aggregate_rblineprof(lineprof_stats)
+        samples = []
+
+        lineprof_stats.each do |(file, stats)|
+          source_lines = File.read(file).each_line.to_a
+          line_samples = []
+
+          total_duration = microsec_to_millisec(stats[0][0])
+          total_events   = stats[0][2]
+
+          next if total_duration <= MINIMUM_DURATION
+
+          stats[1..-1].each_with_index do |data, index|
+            next unless source_lines[index]
+
+            duration = microsec_to_millisec(data[0])
+            events   = data[2]
+
+            line_samples << LineSample.new(duration, events)
+          end
+
+          samples << FileSample.
+            new(file, line_samples, total_duration, total_events)
+        end
+
+        samples
+      end
+
+      def microsec_to_millisec(microsec)
+        microsec / 1000.0
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb
new file mode 100644
index 00000000000..38df7a88e4e
--- /dev/null
+++ b/lib/gitlab/sherlock/line_sample.rb
@@ -0,0 +1,20 @@
+module Gitlab
+  module Sherlock
+    class LineSample
+      attr_reader :duration, :events
+
+      def initialize(duration, events)
+        @duration = duration
+        @events = events
+      end
+
+      def percentage_of(total_duration)
+        (duration.to_f / total_duration) * 100.0
+      end
+
+      def majority_of?(total_duration)
+        percentage_of(total_duration) >= 30
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb
new file mode 100644
index 00000000000..8c0b77dce1a
--- /dev/null
+++ b/lib/gitlab/sherlock/location.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Sherlock
+    class Location
+      attr_reader :path, :line
+
+      SHERLOCK_DIR = File.dirname(__FILE__)
+
+      def self.from_ruby_location(location)
+        new(location.path, location.lineno)
+      end
+
+      def initialize(path, line)
+        @path = path
+        @line = line
+      end
+
+      def application?
+        @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb
new file mode 100644
index 00000000000..fca7be858eb
--- /dev/null
+++ b/lib/gitlab/sherlock/middleware.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Sherlock
+    # Rack middleware used for tracking request metrics.
+    class Middleware
+      CONTENT_TYPES = /text\/html|application\/json/i
+
+      IGNORE_PATHS = %r{^/sherlock}
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if instrument?(env)
+          call_with_instrumentation(env)
+        else
+          @app.call(env)
+        end
+      end
+
+      def call_with_instrumentation(env)
+        trans = Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI'])
+        retval = trans.run { @app.call(env) }
+
+        Sherlock.collection.add(trans)
+
+        retval
+      end
+
+      def instrument?(env)
+        !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES &&
+           env['REQUEST_URI'] !~ IGNORE_PATHS)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb
new file mode 100644
index 00000000000..af76e6fd2bf
--- /dev/null
+++ b/lib/gitlab/sherlock/query.rb
@@ -0,0 +1,96 @@
+module Gitlab
+  module Sherlock
+    class Query
+      attr_reader :id, :query, :started_at, :finished_at, :backtrace
+
+      PREFIX_NEWLINE = /
+        \s+(FROM
+          |(LEFT|RIGHT)?INNER\s+JOIN
+          |(LEFT|RIGHT)?OUTER\s+JOIN
+          |WHERE
+          |AND
+          |GROUP\s+BY
+          |ORDER\s+BY
+          |LIMIT
+          |OFFSET)\s+
+      /ix
+
+      def self.new_with_bindings(query, bindings, started_at, finished_at)
+        bindings.each_with_index do |(column, value), index|
+          quoted_value = ActiveRecord::Base.connection.quote(value)
+
+          query = query.gsub("$#{index + 1}", quoted_value)
+        end
+
+        new(query, started_at, finished_at)
+      end
+
+      def initialize(query, started_at, finished_at)
+        @id = SecureRandom.uuid
+        @query = query
+        @started_at = started_at
+        @finished_at = finished_at
+        @backtrace = caller_locations.map do |loc|
+          Location.from_ruby_location(loc)
+        end
+
+        unless @query.end_with?(';')
+          @query += ';'
+        end
+      end
+
+      def duration
+        @duration ||= (@finished_at - @started_at) * 1000.0
+      end
+
+      def to_param
+        @id
+      end
+
+      def formatted_query
+        @formatted_query ||= format_sql(@query)
+      end
+
+      def last_application_frame
+        @last_application_frame ||= @backtrace.find(&:application?)
+      end
+
+      def application_backtrace
+        @application_backtrace ||= @backtrace.select(&:application?)
+      end
+
+      def explain
+        unless @explain
+          ActiveRecord::Base.connection.transaction do
+            @explain = raw_explain(@query).values.flatten.join("\n")
+
+            # Roll back any queries that mutate data so we don't mess up
+            # anything when running explain on an INSERT, UPDATE, DELETE, etc.
+            raise ActiveRecord::Rollback
+          end
+        end
+
+        @explain
+      end
+
+      private
+
+      def raw_explain(query)
+        if Gitlab::Database.postgresql?
+          explain = "EXPLAIN ANALYZE #{query};"
+        else
+          explain = "EXPLAIN #{query};"
+        end
+
+        ActiveRecord::Base.connection.execute(explain)
+      end
+
+      def format_sql(query)
+        query.each_line.
+          map { |line| line.strip }.
+          join("\n").
+          gsub(PREFIX_NEWLINE) { "\n#{$1} " }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb
new file mode 100644
index 00000000000..5cb3e86aa4e
--- /dev/null
+++ b/lib/gitlab/sherlock/transaction.rb
@@ -0,0 +1,85 @@
+module Gitlab
+  module Sherlock
+    class Transaction
+      attr_reader :id, :type, :path, :queries, :file_samples, :started_at,
+        :finished_at
+
+      def initialize(type, path)
+        @id = SecureRandom.uuid
+        @type = type
+        @path = path
+        @duration = 0
+        @queries = []
+        @file_samples = []
+        @started_at = nil
+        @finished_at = nil
+        @thread = Thread.current
+      end
+
+      def run
+        @started_at = Time.now
+
+        subscriber = subscribe_to_active_record
+
+        retval = profile_lines { yield }
+
+        @finished_at = Time.now
+
+        ActiveSupport::Notifications.unsubscribe(subscriber)
+
+        retval
+      end
+
+      def duration
+        @started_at && @finished_at ? @finished_at - @started_at : 0
+      end
+
+      def to_param
+        @id
+      end
+
+      def sorted_queries
+        @queries.sort { |a, b| b.duration <=> a.duration }
+      end
+
+      def sorted_file_samples
+        @file_samples.sort { |a, b| b.duration <=> a.duration }
+      end
+
+      def find_query(id)
+        @queries.find { |query| query.id == id }
+      end
+
+      def find_file_sample(id)
+        @file_samples.find { |sample| sample.id == id }
+      end
+
+      def track_query(query, bindings, start, finish)
+        @queries << Query.new_with_bindings(query, bindings, start, finish)
+      end
+
+      def profile_lines
+        retval = nil
+
+        if Sherlock.enable_line_profiler?
+          retval, @file_samples = LineProfiler.new.profile { yield }
+        else
+          retval = yield
+        end
+
+        retval
+      end
+
+      def subscribe_to_active_record
+        ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
+          # In case somebody uses a multi-threaded server locally (e.g. Puma) we
+          # _only_ want to track queries that originate from the transaction
+          # thread.
+          next unless Thread.current == @thread
+
+          track_query(data[:sql].strip, data[:binds], start, finish)
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 265ef867fff165643784640d837579ce4fcc2207 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 5 Nov 2015 18:01:05 +0100
Subject: Added specs and source documentation for Sherlock

---
 lib/gitlab/sherlock/collection.rb    |  7 +++++++
 lib/gitlab/sherlock/file_sample.rb   |  4 ++++
 lib/gitlab/sherlock/line_profiler.rb | 38 +++++++++++++++++++++++++++++++++++-
 lib/gitlab/sherlock/line_sample.rb   | 16 +++++++++++++++
 lib/gitlab/sherlock/location.rb      |  4 ++++
 lib/gitlab/sherlock/middleware.rb    |  7 ++++++-
 lib/gitlab/sherlock/query.rb         | 26 ++++++++++++++++++++----
 lib/gitlab/sherlock/transaction.rb   | 29 +++++++++++++++++++++------
 8 files changed, 119 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb
index accdc6469bc..66bd6258521 100644
--- a/lib/gitlab/sherlock/collection.rb
+++ b/lib/gitlab/sherlock/collection.rb
@@ -1,5 +1,10 @@
 module Gitlab
   module Sherlock
+    # A collection of transactions recorded by Sherlock.
+    #
+    # Method calls for this class are synchronized using a mutex to allow
+    # sharing of a single Collection instance between threads (e.g. when using
+    # Puma as a webserver).
     class Collection
       include Enumerable
 
@@ -34,6 +39,8 @@ module Gitlab
         sort { |a, b| b.finished_at <=> a.finished_at }
       end
 
+      private
+
       def synchronize(&block)
         @mutex.synchronize(&block)
       end
diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb
index 7a220de9abc..8a3e1a5e5bf 100644
--- a/lib/gitlab/sherlock/file_sample.rb
+++ b/lib/gitlab/sherlock/file_sample.rb
@@ -3,6 +3,10 @@ module Gitlab
     class FileSample
       attr_reader :id, :file, :line_samples, :events, :duration
 
+      # file - The full path to the file this sample belongs to.
+      # line_samples - An array of LineSample objects.
+      # duration - The total execution time in milliseconds.
+      # events - The total amount of events.
       def initialize(file, line_samples, duration, events)
         @id = SecureRandom.uuid
         @file = file
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
index a191b1e646d..152749dcc39 100644
--- a/lib/gitlab/sherlock/line_profiler.rb
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -1,12 +1,36 @@
 module Gitlab
   module Sherlock
+    # Class for profiling code on a per line basis.
+    #
+    # The LineProfiler class can be used to profile code on per line basis
+    # without littering your code with Ruby implementation specific profiling
+    # methods.
+    #
+    # This profiler only includes samples taking longer than a given threshold
+    # and those that occur in the actual application (e.g. files from Gems are
+    # ignored).
     class LineProfiler
       # The minimum amount of time that has to be spent in a file for it to be
       # included in a list of samples.
       MINIMUM_DURATION = 10.0
 
+      # Profiles the given block.
+      #
+      # Example:
+      #
+      #     profiler = LineProfiler.new
+      #
+      #     retval, samples = profiler.profile do
+      #       "cats are amazing"
+      #     end
+      #
+      #     retval  # => "cats are amazing"
+      #     samples # => [#<Gitlab::Sherlock::FileSample ...>, ...]
+      #
+      # Returns an Array containing the block's return value and an Array of
+      # FileSample objects.
       def profile(&block)
-        if RUBY_ENGINE == 'ruby'
+        if mri?
           profile_mri(&block)
         else
           raise NotImplementedError,
@@ -14,6 +38,7 @@ module Gitlab
         end
       end
 
+      # Profiles the given block using rblineprof (MRI only).
       def profile_mri
         retval  = nil
         samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield }
@@ -24,6 +49,11 @@ module Gitlab
       end
 
       # Returns an Array of file samples based on the output of rblineprof.
+      #
+      # lineprof_stats - A Hash containing rblineprof statistics on a per file
+      #                  basis.
+      #
+      # Returns an Array of FileSample objects.
       def aggregate_rblineprof(lineprof_stats)
         samples = []
 
@@ -52,9 +82,15 @@ module Gitlab
         samples
       end
 
+      private
+
       def microsec_to_millisec(microsec)
         microsec / 1000.0
       end
+
+      def mri?
+        RUBY_ENGINE == 'ruby'
+      end
     end
   end
 end
diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb
index 38df7a88e4e..eb1948eb6d6 100644
--- a/lib/gitlab/sherlock/line_sample.rb
+++ b/lib/gitlab/sherlock/line_sample.rb
@@ -3,15 +3,31 @@ module Gitlab
     class LineSample
       attr_reader :duration, :events
 
+      # duration - The execution time in milliseconds.
+      # events - The amount of events.
       def initialize(duration, events)
         @duration = duration
         @events = events
       end
 
+      # Returns the sample duration percentage relative to the given duration.
+      #
+      # Example:
+      #
+      #     sample.duration            # => 150
+      #     sample.percentage_of(1500) # => 10.0
+      #
+      # total_duration - The total duration to compare with.
+      #
+      # Returns a float
       def percentage_of(total_duration)
         (duration.to_f / total_duration) * 100.0
       end
 
+      # Returns true if the current sample takes up the majority of the given
+      # duration.
+      #
+      # total_duration - The total duration to compare with.
       def majority_of?(total_duration)
         percentage_of(total_duration) >= 30
       end
diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb
index 8c0b77dce1a..5ac265618ad 100644
--- a/lib/gitlab/sherlock/location.rb
+++ b/lib/gitlab/sherlock/location.rb
@@ -5,15 +5,19 @@ module Gitlab
 
       SHERLOCK_DIR = File.dirname(__FILE__)
 
+      # Creates a new Location from a `Thread::Backtrace::Location`.
       def self.from_ruby_location(location)
         new(location.path, location.lineno)
       end
 
+      # path - The full path of the frame as a String.
+      # line - The line number of the frame as a Fixnum.
       def initialize(path, line)
         @path = path
         @line = line
       end
 
+      # Returns true if the current frame originated from the application.
       def application?
         @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR)
       end
diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb
index fca7be858eb..687332fc5fc 100644
--- a/lib/gitlab/sherlock/middleware.rb
+++ b/lib/gitlab/sherlock/middleware.rb
@@ -10,6 +10,7 @@ module Gitlab
         @app = app
       end
 
+      # env - A Hash containing Rack environment details.
       def call(env)
         if instrument?(env)
           call_with_instrumentation(env)
@@ -19,7 +20,7 @@ module Gitlab
       end
 
       def call_with_instrumentation(env)
-        trans = Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI'])
+        trans = transaction_from_env(env)
         retval = trans.run { @app.call(env) }
 
         Sherlock.collection.add(trans)
@@ -31,6 +32,10 @@ module Gitlab
         !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES &&
            env['REQUEST_URI'] !~ IGNORE_PATHS)
       end
+
+      def transaction_from_env(env)
+        Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI'])
+      end
     end
   end
 end
diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb
index af76e6fd2bf..4917c4ae2ac 100644
--- a/lib/gitlab/sherlock/query.rb
+++ b/lib/gitlab/sherlock/query.rb
@@ -3,6 +3,7 @@ module Gitlab
     class Query
       attr_reader :id, :query, :started_at, :finished_at, :backtrace
 
+      # SQL identifiers that should be prefixed with newlines.
       PREFIX_NEWLINE = /
         \s+(FROM
           |(LEFT|RIGHT)?INNER\s+JOIN
@@ -12,11 +13,20 @@ module Gitlab
           |GROUP\s+BY
           |ORDER\s+BY
           |LIMIT
-          |OFFSET)\s+
-      /ix
-
+          |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :<
+
+      # Creates a new Query using a String and a separate Array of bindings.
+      #
+      # query - A String containing a SQL query, optionally with numeric
+      #         placeholders (`$1`, `$2`, etc).
+      #
+      # bindings - An Array of ActiveRecord columns and their values.
+      # started_at - The start time of the query as a Time-like object.
+      # finished_at - The completion time of the query as a Time-like object.
+      #
+      # Returns a new Query object.
       def self.new_with_bindings(query, bindings, started_at, finished_at)
-        bindings.each_with_index do |(column, value), index|
+        bindings.each_with_index do |(_, value), index|
           quoted_value = ActiveRecord::Base.connection.quote(value)
 
           query = query.gsub("$#{index + 1}", quoted_value)
@@ -25,6 +35,9 @@ module Gitlab
         new(query, started_at, finished_at)
       end
 
+      # query - The SQL query as a String (without placeholders).
+      # started_at - The start time of the query as a Time-like object.
+      # finished_at - The completion time of the query as a Time-like object.
       def initialize(query, started_at, finished_at)
         @id = SecureRandom.uuid
         @query = query
@@ -39,6 +52,7 @@ module Gitlab
         end
       end
 
+      # Returns the query duration in milliseconds.
       def duration
         @duration ||= (@finished_at - @started_at) * 1000.0
       end
@@ -47,18 +61,22 @@ module Gitlab
         @id
       end
 
+      # Returns a human readable version of the query.
       def formatted_query
         @formatted_query ||= format_sql(@query)
       end
 
+      # Returns the last application frame of the backtrace.
       def last_application_frame
         @last_application_frame ||= @backtrace.find(&:application?)
       end
 
+      # Returns an Array of application frames (excluding Gems and the likes).
       def application_backtrace
         @application_backtrace ||= @backtrace.select(&:application?)
       end
 
+      # Returns the query plan as a String.
       def explain
         unless @explain
           ActiveRecord::Base.connection.transaction do
diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb
index 5cb3e86aa4e..4641f15ee33 100644
--- a/lib/gitlab/sherlock/transaction.rb
+++ b/lib/gitlab/sherlock/transaction.rb
@@ -4,11 +4,12 @@ module Gitlab
       attr_reader :id, :type, :path, :queries, :file_samples, :started_at,
         :finished_at
 
+      # type - The type of transaction (e.g. "GET", "POST", etc)
+      # path - The path of the transaction (e.g. the HTTP request path)
       def initialize(type, path)
         @id = SecureRandom.uuid
         @type = type
         @path = path
-        @duration = 0
         @queries = []
         @file_samples = []
         @started_at = nil
@@ -16,6 +17,7 @@ module Gitlab
         @thread = Thread.current
       end
 
+      # Runs the transaction and returns the block's return value.
       def run
         @started_at = Time.now
 
@@ -30,34 +32,43 @@ module Gitlab
         retval
       end
 
+      # Returns the duration in seconds.
       def duration
-        @started_at && @finished_at ? @finished_at - @started_at : 0
+        @duration ||= started_at && finished_at ? finished_at - started_at : 0
       end
 
       def to_param
         @id
       end
 
+      # Returns the queries sorted in descending order by their durations.
       def sorted_queries
         @queries.sort { |a, b| b.duration <=> a.duration }
       end
 
+      # Returns the file samples sorted in descending order by their durations.
       def sorted_file_samples
         @file_samples.sort { |a, b| b.duration <=> a.duration }
       end
 
+      # Finds a query by the given ID.
+      #
+      # id - The query ID as a String.
+      #
+      # Returns a Query object if one could be found, nil otherwise.
       def find_query(id)
         @queries.find { |query| query.id == id }
       end
 
+      # Finds a file sample by the given ID.
+      #
+      # id - The query ID as a String.
+      #
+      # Returns a FileSample object if one could be found, nil otherwise.
       def find_file_sample(id)
         @file_samples.find { |sample| sample.id == id }
       end
 
-      def track_query(query, bindings, start, finish)
-        @queries << Query.new_with_bindings(query, bindings, start, finish)
-      end
-
       def profile_lines
         retval = nil
 
@@ -70,6 +81,12 @@ module Gitlab
         retval
       end
 
+      private
+
+      def track_query(query, bindings, start, finish)
+        @queries << Query.new_with_bindings(query, bindings, start, finish)
+      end
+
       def subscribe_to_active_record
         ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
           # In case somebody uses a multi-threaded server locally (e.g. Puma) we
-- 
cgit v1.2.1


From 7b5fd8742e6112491f61f27dcca2d8e441cc33a1 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 9 Nov 2015 12:36:01 +0100
Subject: Track the amount of times views are rendered

---
 lib/gitlab/sherlock/transaction.rb | 57 ++++++++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb
index 4641f15ee33..d87a4c9bb4a 100644
--- a/lib/gitlab/sherlock/transaction.rb
+++ b/lib/gitlab/sherlock/transaction.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Sherlock
     class Transaction
       attr_reader :id, :type, :path, :queries, :file_samples, :started_at,
-        :finished_at
+        :finished_at, :view_counts
 
       # type - The type of transaction (e.g. "GET", "POST", etc)
       # path - The path of the transaction (e.g. the HTTP request path)
@@ -15,20 +15,19 @@ module Gitlab
         @started_at = nil
         @finished_at = nil
         @thread = Thread.current
+        @view_counts = Hash.new(0)
       end
 
       # Runs the transaction and returns the block's return value.
       def run
         @started_at = Time.now
 
-        subscriber = subscribe_to_active_record
-
-        retval = profile_lines { yield }
+        retval = with_subscriptions do
+          profile_lines { yield }
+        end
 
         @finished_at = Time.now
 
-        ActiveSupport::Notifications.unsubscribe(subscriber)
-
         retval
       end
 
@@ -81,21 +80,51 @@ module Gitlab
         retval
       end
 
+      def subscribe_to_active_record
+        ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
+          next unless same_thread?
+
+          track_query(data[:sql].strip, data[:binds], start, finish)
+        end
+      end
+
+      def subscribe_to_action_view
+        regex = /render_(template|partial)\.action_view/
+
+        ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data|
+          next unless same_thread?
+
+          track_view(data[:identifier])
+        end
+      end
+
       private
 
       def track_query(query, bindings, start, finish)
         @queries << Query.new_with_bindings(query, bindings, start, finish)
       end
 
-      def subscribe_to_active_record
-        ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
-          # In case somebody uses a multi-threaded server locally (e.g. Puma) we
-          # _only_ want to track queries that originate from the transaction
-          # thread.
-          next unless Thread.current == @thread
+      def track_view(path)
+        @view_counts[path] += 1
+      end
 
-          track_query(data[:sql].strip, data[:binds], start, finish)
-        end
+      def with_subscriptions
+        ar_subscriber = subscribe_to_active_record
+        av_subscriber = subscribe_to_action_view
+
+        retval = yield
+
+        ActiveSupport::Notifications.unsubscribe(ar_subscriber)
+        ActiveSupport::Notifications.unsubscribe(av_subscriber)
+
+        retval
+      end
+
+      # In case somebody uses a multi-threaded server locally (e.g. Puma) we
+      # _only_ want to track notifications that originate from the transaction
+      # thread.
+      def same_thread?
+        Thread.current == @thread
       end
     end
   end
-- 
cgit v1.2.1


From 73cf0f1647806a4ce064707c6f1f416181de48ef Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 9 Nov 2015 15:54:13 +0100
Subject: Only load rblineprof when actually needed

This ensures the application can still boot when the "development" group
is not available.
---
 lib/gitlab/sherlock.rb               | 1 -
 lib/gitlab/sherlock/line_profiler.rb | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb
index c4b35b24ceb..6360527a7aa 100644
--- a/lib/gitlab/sherlock.rb
+++ b/lib/gitlab/sherlock.rb
@@ -1,5 +1,4 @@
 require 'securerandom'
-require 'rblineprof' if RUBY_ENGINE == 'ruby'
 
 module Gitlab
   module Sherlock
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
index 152749dcc39..aa1468bff6b 100644
--- a/lib/gitlab/sherlock/line_profiler.rb
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -40,6 +40,8 @@ module Gitlab
 
       # Profiles the given block using rblineprof (MRI only).
       def profile_mri
+        require 'rblineprof'
+
         retval  = nil
         samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield }
 
-- 
cgit v1.2.1


From 1b7a2fc5363d7e2334f5c48940e8eca9d88354a6 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Tue, 10 Nov 2015 11:49:38 +0100
Subject: Improve Continuous Integration graphs page

* fix commit duration graph
* make graphs responsive
* fix wrong padding
* add a bit of explanation to colors

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/ci/charts.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index 915a4f526a6..5ff7407c6fe 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -60,7 +60,8 @@ module Ci
 
     class BuildTime < Chart
       def collect
-        commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30)
+        commits = project.commits.last(30)
+
         commits.each do |commit|
           @labels << commit.short_sha
           @build_times << (commit.duration / 60)
-- 
cgit v1.2.1


From d0e3e823a2dd56260550aec648b0cbfae64543ae Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 12 Oct 2015 23:47:32 +0200
Subject: Implement Build Artifacts

- Offloads uploading to GitLab Workhorse
- Use /authorize request for fast uploading
- Added backup recipes for artifacts
- Support download acceleration using X-Sendfile
---
 lib/api/helpers.rb                 |  44 ++++++++++++++++
 lib/backup/artifacts.rb            |  13 +++++
 lib/backup/manager.rb              |   2 +-
 lib/ci/api/api.rb                  |   1 +
 lib/ci/api/builds.rb               | 102 +++++++++++++++++++++++++++++++++++++
 lib/ci/api/entities.rb             |   7 +++
 lib/ci/api/helpers.rb              |  11 ++++
 lib/ci/gitlab_ci_yaml_processor.rb |   9 +++-
 lib/file_streamer.rb               |  16 ++++++
 lib/gitlab/current_settings.rb     |   1 +
 lib/support/nginx/gitlab           |  16 ++++++
 lib/support/nginx/gitlab-ssl       |  16 ++++++
 lib/tasks/gitlab/backup.rake       |  21 ++++++++
 lib/uploaded_file.rb               |  37 ++++++++++++++
 14 files changed, 293 insertions(+), 3 deletions(-)
 create mode 100644 lib/backup/artifacts.rb
 create mode 100644 lib/file_streamer.rb
 create mode 100644 lib/uploaded_file.rb

(limited to 'lib')

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 652bdf9b278..b980cd8391e 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -133,6 +133,12 @@ module API
       authorize! :admin_project, user_project
     end
 
+    def require_gitlab_workhorse!
+      unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present?
+        forbidden!('Request should be executed via GitLab Workhorse')
+      end
+    end
+
     def can?(object, action, subject)
       abilities.allowed?(object, action, subject)
     end
@@ -234,6 +240,10 @@ module API
       render_api_error!(message || '409 Conflict', 409)
     end
 
+    def file_to_large!
+      render_api_error!('413 Request Entity Too Large', 413)
+    end
+
     def render_validation_error!(model)
       if model.errors.any?
         render_api_error!(model.errors.messages || '400 Bad Request', 400)
@@ -282,6 +292,40 @@ module API
       end
     end
 
+    # file helpers
+
+    def uploaded_file!(uploads_path)
+      required_attributes! [:file]
+
+      # sanitize file paths
+      # this requires for all paths to exist
+      uploads_path = File.realpath(uploads_path)
+      file_path = File.realpath(params[:file])
+      bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+
+      UploadedFile.new(
+        file_path,
+        params[:filename],
+        params[:filetype] || 'application/octet-stream',
+      )
+    end
+
+    def present_file!(path, filename, content_type = 'application/octet-stream')
+      filename ||= File.basename(path)
+      header['Content-Disposition'] = "attachment; filename=#{filename}"
+      header['Content-Transfer-Encoding'] = 'binary'
+      content_type content_type
+
+      # Support download acceleration
+      case headers['X-Sendfile-Type']
+      when 'X-Sendfile'
+        header['X-Sendfile'] = path
+        body
+      else
+        file FileStreamer.new(path)
+      end
+    end
+
     private
 
     def add_pagination_headers(paginated, per_page)
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
new file mode 100644
index 00000000000..51fa3867e67
--- /dev/null
+++ b/lib/backup/artifacts.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Artifacts < Files
+    def initialize
+      super('artifacts', ArtifactUploader.artifacts_path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index f011fd03de0..9e15d5411a1 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,7 +150,7 @@ module Backup
     private
 
     def backup_contents
-      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
     end
 
     def folders_to_backup
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 0a4cbf69b63..07e68216d7f 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -27,6 +27,7 @@ module Ci
 
       helpers Helpers
       helpers ::API::Helpers
+      helpers Gitlab::CurrentSettings
 
       mount Builds
       mount Commits
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 83ca1e6481c..622849c4b11 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -47,6 +47,108 @@ module Ci
             build.drop
           end
         end
+
+        # Authorize artifacts uploading for build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        #   size (optional) - the size of uploaded file
+        # Example Request:
+        #   POST /builds/:id/artifacts/authorize
+        post ":id/artifacts/authorize" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          if params[:filesize]
+            file_size = params[:filesize].to_i
+            file_to_large! unless file_size < max_artifacts_size
+          end
+
+          status 200
+          { temp_path: ArtifactUploader.artifacts_upload_path }
+        end
+
+        # Upload artifacts to build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   Content-Type - File content type
+        #   Content-Disposition - File media type and real name
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Body:
+        #   The file content
+        #
+        # Parameters (set by GitLab Workhorse):
+        #   file - path to locally stored body (generated by Workhorse)
+        #   filename - real filename as send in Content-Disposition
+        #   filetype - real content type as send in Content-Type
+        #   filesize - real file size as send in Content-Length
+        # Example Request:
+        #   POST /builds/:id/artifacts
+        post ":id/artifacts" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          file = uploaded_file!(ArtifactUploader.artifacts_upload_path)
+          file_to_large! unless file.size < max_artifacts_size
+
+          if build.update_attributes(artifacts_file: file)
+            present build, with: Entities::Build
+          else
+            render_validation_error!(build)
+          end
+        end
+
+        # Download the artifacts file from build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   GET /builds/:id/artifacts
+        get ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          artifacts_file = build.artifacts_file
+
+          unless artifacts_file.file_storage?
+            return redirect_to build.artifacts_file.url
+          end
+
+          unless artifacts_file.exists?
+            not_found!
+          end
+
+          present_file!(artifacts_file.path, artifacts_file.filename)
+        end
+
+        # Remove the artifacts file from build
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   DELETE /builds/:id/artifacts
+        delete ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          build.remove_artifacts_file!
+        end
       end
     end
   end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b80c0b8b273..750f421872d 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -11,10 +11,16 @@ module Ci
         expose :builds
       end
 
+      class ArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
       class Build < Grape::Entity
         expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
           :before_sha, :allow_git_fetch, :project_name
 
+        expose :name, :token, :stage
+
         expose :options do |model|
           model.options
         end
@@ -24,6 +30,7 @@ module Ci
         end
 
         expose :variables
+        expose :artifacts_file, using: ArtifactFile
       end
 
       class Runner < Grape::Entity
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 7e4986b6af3..02502333756 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -1,6 +1,8 @@
 module Ci
   module API
     module Helpers
+      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+      BUILD_TOKEN_PARAM = :token
       UPDATE_RUNNER_EVERY = 60
 
       def authenticate_runners!
@@ -15,6 +17,11 @@ module Ci
         forbidden! unless project.valid_token?(params[:project_token])
       end
 
+      def authenticate_build_token!(build)
+        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+        forbidden! unless token && build.valid_token?(token)
+      end
+
       def update_runner_last_contact
         # Use a random threshold to prevent beating DB updates
         contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
@@ -32,6 +39,10 @@ module Ci
         info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
         current_runner.update(info)
       end
+
+      def max_artifacts_size
+        current_application_settings.max_artifacts_size.megabytes.to_i
+      end
     end
   end
 end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 0f57a4f53ab..6f9af5388ca 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -5,7 +5,7 @@ module Ci
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
     ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
+    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
 
     attr_reader :before_script, :image, :services, :variables, :path
 
@@ -77,7 +77,8 @@ module Ci
         when: job[:when] || 'on_success',
         options: {
           image: job[:image] || @image,
-          services: job[:services] || @services
+          services: job[:services] || @services,
+          artifacts: job[:artifacts]
         }.compact
       }
     end
@@ -159,6 +160,10 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
+      if job[:artifacts] && !validate_array_of_strings(job[:artifacts])
+        raise ValidationError, "#{name}: artifacts parameter should be an array of strings"
+      end
+
       if job[:allow_failure] && !job[:allow_failure].in?([true, false])
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb
new file mode 100644
index 00000000000..4e3c6d3c773
--- /dev/null
+++ b/lib/file_streamer.rb
@@ -0,0 +1,16 @@
+class FileStreamer #:nodoc:
+  attr_reader :to_path
+
+  def initialize(path)
+    @to_path = path
+  end
+
+  # Stream the file's contents if Rack::Sendfile isn't present.
+  def each
+    File.open(to_path, 'rb') do |file|
+      while chunk = file.read(16384)
+        yield chunk
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index cd84afa31d5..2d3e32d9539 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -25,6 +25,7 @@ module Gitlab
         session_expire_delay: Settings.gitlab['session_expire_delay'],
         import_sources: Settings.gitlab['import_sources'],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+        max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
       )
     end
 
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index e767027dc29..e511d5e4b4b 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -131,6 +131,22 @@ server {
     return 418;
   }
 
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+      error_page 418 = @gitlab-git-http-server;
+      return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+      error_page 418 = @gitlab-git-http-server;
+      return 418;
+  }
+
   location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 4d31e31f8d5..47b1ec8cb0c 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -178,6 +178,22 @@ server {
     return 418;
   }
 
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+      error_page 418 = @gitlab-git-http-server;
+      return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+      error_page 418 = @gitlab-git-http-server;
+      return 418;
+  }
+
   location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index f20c7f71ba5..3c46bcea40e 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -12,6 +12,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:create"].invoke
       Rake::Task["gitlab:backup:uploads:create"].invoke
       Rake::Task["gitlab:backup:builds:create"].invoke
+      Rake::Task["gitlab:backup:artifacts:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -32,6 +33,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
       Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
+      Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -113,6 +115,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :artifacts do
+      task create: :environment do
+        $progress.puts "Dumping artifacts ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Artifacts.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring artifacts ... ".blue
+        Backup::Artifacts.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     def configure_cron_mode
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
new file mode 100644
index 00000000000..d4291f012d3
--- /dev/null
+++ b/lib/uploaded_file.rb
@@ -0,0 +1,37 @@
+require "tempfile"
+require "fileutils"
+
+# Taken from: Rack::Test::UploadedFile
+class UploadedFile
+
+  # The filename, *not* including the path, of the "uploaded" file
+  attr_reader :original_filename
+
+  # The tempfile
+  attr_reader :tempfile
+
+  # The content type of the "uploaded" file
+  attr_accessor :content_type
+
+  def initialize(path, filename, content_type = "text/plain")
+    raise "#{path} file does not exist" unless ::File.exist?(path)
+
+    @content_type = content_type
+    @original_filename = filename || ::File.basename(path)
+    @tempfile = File.new(path, 'rb')
+  end
+
+  def path
+    @tempfile.path
+  end
+
+  alias_method :local_path, :path
+
+  def method_missing(method_name, *args, &block) #:nodoc:
+    @tempfile.__send__(method_name, *args, &block)
+  end
+
+  def respond_to?(method_name, include_private = false) #:nodoc:
+    @tempfile.respond_to?(method_name, include_private) || super
+  end
+end
-- 
cgit v1.2.1


From 97f58bae87dfcfb36d5a7a490b1c0983435a19f4 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 12:18:00 +0100
Subject: Change artifacts syntax to allow uploading untracked files

---
 lib/ci/gitlab_ci_yaml_processor.rb | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 6f9af5388ca..2e2209031ee 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -160,11 +160,17 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
-      if job[:artifacts] && !validate_array_of_strings(job[:artifacts])
-        raise ValidationError, "#{name}: artifacts parameter should be an array of strings"
+      if job[:artifacts]
+        if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
+          raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
+        end
+
+        if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
+          raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
+        end
       end
 
-      if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
 
@@ -187,6 +193,10 @@ module Ci
       value.is_a?(String) || value.is_a?(Symbol)
     end
 
+    def validate_boolean(value)
+      value.in?([true, false])
+    end
+
     def process?(only_params, except_params, ref, tag)
       if only_params.present?
         return false unless matching?(only_params, ref, tag)
-- 
cgit v1.2.1


From 445cdb7579792d0d76c2562b971583bd7d05429b Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 12:49:47 +0100
Subject: Move tmp artifacts to shared/artifacts/tmp/. Check for
 GitLab-Workhorse now

---
 lib/api/helpers.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index b980cd8391e..077537959d7 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -134,7 +134,7 @@ module API
     end
 
     def require_gitlab_workhorse!
-      unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present?
+      unless env['HTTP_GITLAB_WORKHORSE'].present?
         forbidden!('Request should be executed via GitLab Workhorse')
       end
     end
-- 
cgit v1.2.1


From 7e4e3fb3b6177d2863580786e9ef6ee3a4ccf037 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 12:55:32 +0100
Subject: Fix nginx config to use @gitlab-workhorse

---
 lib/support/nginx/gitlab     | 8 ++++----
 lib/support/nginx/gitlab-ssl | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index e511d5e4b4b..0a7a4118077 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -134,16 +134,16 @@ server {
   # Build artifacts should be submitted to this location
   location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
       client_max_body_size 0;
-      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-      error_page 418 = @gitlab-git-http-server;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
       return 418;
   }
 
   # Build artifacts should be submitted to this location
   location ~ /ci/api/v1/builds/[0-9]+/artifacts {
       client_max_body_size 0;
-      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-      error_page 418 = @gitlab-git-http-server;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
       return 418;
   }
 
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 47b1ec8cb0c..b463d5b6aa9 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -181,16 +181,16 @@ server {
   # Build artifacts should be submitted to this location
   location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
       client_max_body_size 0;
-      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-      error_page 418 = @gitlab-git-http-server;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
       return 418;
   }
 
   # Build artifacts should be submitted to this location
   location ~ /ci/api/v1/builds/[0-9]+/artifacts {
       client_max_body_size 0;
-      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-      error_page 418 = @gitlab-git-http-server;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
       return 418;
   }
 
-- 
cgit v1.2.1


From db3213fc1c653b20783f9a41074eaf17132010de Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 22:01:26 +0100
Subject: Use normal file upload mechanism to upload artifacts

---
 lib/api/helpers.rb   | 16 ++++++++++------
 lib/ci/api/builds.rb | 16 +++++++---------
 2 files changed, 17 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 077537959d7..92540ccf2b1 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -294,19 +294,23 @@ module API
 
     # file helpers
 
-    def uploaded_file!(uploads_path)
-      required_attributes! [:file]
+    def uploaded_file!(field, uploads_path)
+      if params[field]
+        bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
+        return params[field]
+      end
 
       # sanitize file paths
-      # this requires for all paths to exist
+      # this requires all paths to exist
+      required_attributes! %W(#{field}.path)
       uploads_path = File.realpath(uploads_path)
-      file_path = File.realpath(params[:file])
+      file_path = File.realpath(params["#{field}.path"])
       bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
 
       UploadedFile.new(
         file_path,
-        params[:filename],
-        params[:filetype] || 'application/octet-stream',
+        params["#{field}.name"],
+        params["#{field}.type"] || 'application/octet-stream',
       )
     end
 
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 622849c4b11..dab0df12635 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -53,7 +53,7 @@ module Ci
         # Parameters:
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
-        #   size (optional) - the size of uploaded file
+        #   filesize (optional) - the size of uploaded file
         # Example Request:
         #   POST /builds/:id/artifacts/authorize
         post ":id/artifacts/authorize" do
@@ -77,18 +77,16 @@ module Ci
         # Parameters:
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
+        #   file (required) - The uploaded file
+        # Parameters (accelerated by GitLab Workhorse):
+        #   file.path - path to locally stored body (generated by Workhorse)
+        #   file.name - real filename as send in Content-Disposition
+        #   file.type - real content type as send in Content-Type
         # Headers:
-        #   Content-Type - File content type
-        #   Content-Disposition - File media type and real name
         #   BUILD-TOKEN (required) - The build authorization token, the same as token
         # Body:
         #   The file content
         #
-        # Parameters (set by GitLab Workhorse):
-        #   file - path to locally stored body (generated by Workhorse)
-        #   filename - real filename as send in Content-Disposition
-        #   filetype - real content type as send in Content-Type
-        #   filesize - real file size as send in Content-Length
         # Example Request:
         #   POST /builds/:id/artifacts
         post ":id/artifacts" do
@@ -98,7 +96,7 @@ module Ci
           authenticate_build_token!(build)
           forbidden!('build is not running') unless build.running?
 
-          file = uploaded_file!(ArtifactUploader.artifacts_upload_path)
+          file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
           file_to_large! unless file.size < max_artifacts_size
 
           if build.update_attributes(artifacts_file: file)
-- 
cgit v1.2.1


From 58429d9b26b2f2b62fecab012fce0ebe36571129 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Tue, 10 Nov 2015 18:34:05 +0100
Subject: Add method complexity check to CI

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/flog.rake | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 lib/tasks/flog.rake

(limited to 'lib')

diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake
new file mode 100644
index 00000000000..4cb19c0c937
--- /dev/null
+++ b/lib/tasks/flog.rake
@@ -0,0 +1,24 @@
+desc 'Code complexity analyze via flog'
+task :flog do
+  output = %x(bundle exec flog -m app/ lib/gitlab)
+  exit_code = 0
+  output = output.lines
+
+  # Skip total complexity score
+  output.shift
+
+  # Skip some trash info
+  output.shift
+
+  output.each do |line|
+    score, method = line.split(" ")
+    score = score.to_i
+
+    if score > 40
+      exit_code = 1
+      puts "High complexity in #{method}. Score: #{score}"
+    end
+  end
+
+  exit exit_code
+end
-- 
cgit v1.2.1


From eef129bd4ad41665cf5c1f1631e80b2f1bb3e4af Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 10 Nov 2015 22:21:15 +0100
Subject: Final fixes

---
 lib/ci/api/builds.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index dab0df12635..0a586672807 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -69,7 +69,7 @@ module Ci
           end
 
           status 200
-          { temp_path: ArtifactUploader.artifacts_upload_path }
+          { TempPath: ArtifactUploader.artifacts_upload_path }
         end
 
         # Upload artifacts to build - Runners only
-- 
cgit v1.2.1


From 6748dd2fd0b2cf694661e910c6c1bb6ac2be69a1 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Wed, 11 Nov 2015 15:30:23 +0100
Subject: Allow flog failure for now

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/flog.rake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake
index 4cb19c0c937..3bfe999ae74 100644
--- a/lib/tasks/flog.rake
+++ b/lib/tasks/flog.rake
@@ -2,6 +2,7 @@ desc 'Code complexity analyze via flog'
 task :flog do
   output = %x(bundle exec flog -m app/ lib/gitlab)
   exit_code = 0
+  minimum_score = 70
   output = output.lines
 
   # Skip total complexity score
@@ -14,7 +15,7 @@ task :flog do
     score, method = line.split(" ")
     score = score.to_i
 
-    if score > 40
+    if score > minimum_score
       exit_code = 1
       puts "High complexity in #{method}. Score: #{score}"
     end
-- 
cgit v1.2.1


From 98c954ee815823f27cf021b34684c2c3d5bb0917 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Wed, 11 Nov 2015 16:29:00 +0100
Subject: Add flay: tool to find duplicate code

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/flay.rake | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 lib/tasks/flay.rake

(limited to 'lib')

diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
new file mode 100644
index 00000000000..5efffc2cdac
--- /dev/null
+++ b/lib/tasks/flay.rake
@@ -0,0 +1,9 @@
+desc 'Code duplication analyze via flay'
+task :flay do
+  output = %x(bundle exec flay app/ lib/gitlab/)
+
+  if output.include? "Similar code found"
+    puts output
+    exit 1
+  end
+end
-- 
cgit v1.2.1