From 16f8ca566b8637dc8092a6b630c23a82a905b437 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Thu, 1 Oct 2015 23:40:29 -0400
Subject: Add custom protocol whitelisting to SanitizationFilter

Addresses internal https://dev.gitlab.org/gitlab/gitlabhq/issues/2613
---
 lib/gitlab/markdown/sanitization_filter.rb | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb
index e368de7d848..ffb9dc33b64 100644
--- a/lib/gitlab/markdown/sanitization_filter.rb
+++ b/lib/gitlab/markdown/sanitization_filter.rb
@@ -48,6 +48,12 @@ module Gitlab
         # Allow span elements
         whitelist[:elements].push('span')
 
+        # Allow any protocol in `a` elements...
+        whitelist[:protocols].delete('a')
+
+        # ...but then remove links with the `javascript` protocol
+        whitelist[:transformers].push(remove_javascript_links)
+
         # Remove `rel` attribute from `a` elements
         whitelist[:transformers].push(remove_rel)
 
@@ -57,6 +63,19 @@ module Gitlab
         whitelist
       end
 
+      def remove_javascript_links
+        lambda do |env|
+          node = env[:node]
+
+          return unless node.name == 'a'
+          return unless node.has_attribute?('href')
+
+          if node['href'].start_with?('javascript', ':javascript')
+            node.remove_attribute('href')
+          end
+        end
+      end
+
       def remove_rel
         lambda do |env|
           if env[:node_name] == 'a'
-- 
cgit v1.2.1


From d3734fbd89c069d35856b440f12109af8a7ef9c9 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 6 Oct 2015 14:43:19 +0200
Subject: Use tar for intermediate backup storage

During the backup we create an intermediate copy of two directories:
builds and uploads. Instead of creating many small files with 'cp
-r', we now use tar (and fast gzip) to create single intermediate
files. This saves on disk IO and disk space while creating a backup.
---
 lib/backup/builds.rb  | 31 ++-----------------------------
 lib/backup/files.rb   | 39 +++++++++++++++++++++++++++++++++++++++
 lib/backup/manager.rb |  4 ++--
 lib/backup/uploads.rb | 30 ++----------------------------
 4 files changed, 45 insertions(+), 59 deletions(-)
 create mode 100644 lib/backup/files.rb

(limited to 'lib')

diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
index 6f56f680bb9..d269f8e260c 100644
--- a/lib/backup/builds.rb
+++ b/lib/backup/builds.rb
@@ -1,34 +1,7 @@
 module Backup
-  class Builds
-    attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
-
+  class Builds < Files
     def initialize
-      @app_builds_dir = Settings.gitlab_ci.builds_path
-      @backup_dir = Gitlab.config.backup.path
-      @backup_builds_dir = File.join(Gitlab.config.backup.path, 'builds')
-    end
-
-    # Copy builds from builds directory to backup/builds
-    def dump
-      FileUtils.rm_rf(backup_builds_dir)
-      # Ensure the parent dir of backup_builds_dir exists
-      FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create backup_builds_dir before us
-      FileUtils.mkdir(backup_builds_dir, mode: 0700)
-      FileUtils.cp_r(app_builds_dir, backup_dir)
-    end
-
-    def restore
-      backup_existing_builds_dir
-
-      FileUtils.cp_r(backup_builds_dir, app_builds_dir)
-    end
-
-    def backup_existing_builds_dir
-      timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
-      if File.exists?(app_builds_dir)
-        FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
-      end
+      super(Settings.gitlab_ci.builds_path)
     end
   end
 end
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
new file mode 100644
index 00000000000..d0a6e8f27be
--- /dev/null
+++ b/lib/backup/files.rb
@@ -0,0 +1,39 @@
+require 'open3'
+
+module Backup
+  class Files
+    attr_reader :name, :app_files_dir, :backup_tarball, :backup_dir, :files_parent_dir
+
+    def initialize(app_files_dir)
+      @app_files_dir = File.realpath(app_files_dir)
+      @name = File.basename(app_files_dir)
+      @files_parent_dir = File.realpath(File.join(@app_files_dir, '..'))
+      @backup_dir = Gitlab.config.backup.path
+      @backup_tarball = File.join(@backup_dir, name + '.tar.gz')
+    end
+
+    # Copy files from public/files to backup/files
+    def dump
+      FileUtils.mkdir_p(Gitlab.config.backup.path)
+      run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+    end
+
+    def restore
+      backup_existing_files_dir
+
+      run_pipeline!([%W(gzip -cd), %W(tar -C #{files_parent_dir} -xf -)], in: backup_tarball)
+    end
+
+    def backup_existing_files_dir
+      timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}")
+      if File.exists?(app_files_dir)
+        FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path))
+      end
+    end
+
+    def run_pipeline!(cmd_list, options={})
+      status_list = Open3.pipeline(*cmd_list, options)
+      abort 'Backup failed' unless status_list.compact.all?(&:success?)
+    end
+  end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 5c42f25f4a2..f011fd03de0 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,11 +150,11 @@ module Backup
     private
 
     def backup_contents
-      folders_to_backup + ["backup_information.yml"]
+      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
     end
 
     def folders_to_backup
-      folders = %w{repositories db uploads builds}
+      folders = %w{repositories db}
 
       if ENV["SKIP"]
         return folders.reject{ |folder| ENV["SKIP"].include?(folder) }
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index 1f9626644e6..7c0838cc8b7 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -1,34 +1,8 @@
 module Backup
-  class Uploads
-    attr_reader :app_uploads_dir, :backup_uploads_dir, :backup_dir
+  class Uploads < Files
 
     def initialize
-      @app_uploads_dir = File.realpath(Rails.root.join('public', 'uploads'))
-      @backup_dir = Gitlab.config.backup.path
-      @backup_uploads_dir = File.join(Gitlab.config.backup.path, 'uploads')
-    end
-
-    # Copy uploads from public/uploads to backup/uploads
-    def dump
-      FileUtils.rm_rf(backup_uploads_dir)
-      # Ensure the parent dir of backup_uploads_dir exists
-      FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create backup_uploads_dir before us
-      FileUtils.mkdir(backup_uploads_dir, mode: 0700)
-      FileUtils.cp_r(app_uploads_dir, backup_dir)
-    end
-
-    def restore
-      backup_existing_uploads_dir
-
-      FileUtils.cp_r(backup_uploads_dir, app_uploads_dir)
-    end
-
-    def backup_existing_uploads_dir
-      timestamped_uploads_path = File.join(app_uploads_dir, '..', "uploads.#{Time.now.to_i}")
-      if File.exists?(app_uploads_dir)
-        FileUtils.mv(app_uploads_dir, File.expand_path(timestamped_uploads_path))
-      end
+      super(Rails.root.join('public/uploads'))
     end
   end
 end
-- 
cgit v1.2.1


From 90ddf140b9390647002771572d0375da0bb9dfa4 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 6 Oct 2015 15:06:15 +0200
Subject: Reduce disk IO during SQL backup

By using light gzip compression we can save a lot of disk IO during
the backup.
---
 lib/backup/database.rb | 53 +++++++++++++++++++++++---------------------------
 lib/backup/manager.rb  |  4 ++--
 2 files changed, 26 insertions(+), 31 deletions(-)

(limited to 'lib')

diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 959ac4b7868..4bdf6e1c628 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -2,26 +2,27 @@ require 'yaml'
 
 module Backup
   class Database
-    attr_reader :config, :db_dir
+    attr_reader :config, :db_file_name
 
     def initialize
       @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
-      @db_dir = File.join(Gitlab.config.backup.path, 'db')
+      @db_file_name = File.join(Gitlab.config.backup.path, 'database.sql.gz')
     end
 
     def dump
-      FileUtils.rm_rf(@db_dir)
-      # Ensure the parent dir of @db_dir exists
+      FileUtils.rm_f(db_file_name)
+      compress_rd, compress_wr = IO.pipe
+      compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600])
+      compress_rd.close
+
       FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create @db_dir before us
-      FileUtils.mkdir(@db_dir, mode: 0700)
 
-      success = case config["adapter"]
+      dump_pid = case config["adapter"]
       when /^mysql/ then
         $progress.print "Dumping MySQL database #{config['database']} ... "
         # Workaround warnings from MySQL 5.6 about passwords on cmd line
         ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-        system('mysqldump', *mysql_args, config['database'], out: db_file_name)
+        spawn('mysqldump', *mysql_args, config['database'], out: compress_wr)
       when "postgresql" then
         $progress.print "Dumping PostgreSQL database #{config['database']} ... "
         pg_env
@@ -30,48 +31,42 @@ module Backup
           pgsql_args << "-n"
           pgsql_args << Gitlab.config.backup.pg_schema
         end
-        system('pg_dump', *pgsql_args, config['database'], out: db_file_name)
+        spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
       end
-      report_success(success)
-      abort 'Backup failed' unless success
+      compress_wr.close
+
+      success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? }
 
-      $progress.print 'Compressing database ... '
-      success = system('gzip', db_file_name)
       report_success(success)
-      abort 'Backup failed: compress error' unless success
+      abort 'Backup failed' unless success
     end
 
     def restore
-      $progress.print 'Decompressing database ... '
-      success = system('gzip', '-d', db_file_name_gz)
-      report_success(success)
-      abort 'Restore failed: decompress error' unless success
+      decompress_rd, decompress_wr = IO.pipe
+      decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+      decompress_wr.close
 
-      success = case config["adapter"]
+      restore_pid = case config["adapter"]
       when /^mysql/ then
         $progress.print "Restoring MySQL database #{config['database']} ... "
         # Workaround warnings from MySQL 5.6 about passwords on cmd line
         ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-        system('mysql', *mysql_args, config['database'], in: db_file_name)
+        spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
       when "postgresql" then
         $progress.print "Restoring PostgreSQL database #{config['database']} ... "
         pg_env
-        system('psql', config['database'], '-f', db_file_name)
+        spawn('psql', config['database'], in: decompress_rd)
       end
+      decompress_rd.close
+
+      success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+
       report_success(success)
       abort 'Restore failed' unless success
     end
 
     protected
 
-    def db_file_name
-      File.join(db_dir, 'database.sql')
-    end
-
-    def db_file_name_gz
-      File.join(db_dir, 'database.sql.gz')
-    end
-
     def mysql_args
       args = {
         'host'      => '--host',
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index f011fd03de0..53e79d4d1f7 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,11 +150,11 @@ module Backup
     private
 
     def backup_contents
-      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+      folders_to_backup + ["database.sql.gz", "uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
     end
 
     def folders_to_backup
-      folders = %w{repositories db}
+      folders = %w{repositories}
 
       if ENV["SKIP"]
         return folders.reject{ |folder| ENV["SKIP"].include?(folder) }
-- 
cgit v1.2.1


From 7b71727c562b6f6337a180ae136be94bf0f6ed31 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 6 Oct 2015 15:10:13 +0200
Subject: Remove old "files" tarball explicitly

---
 lib/backup/files.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index d0a6e8f27be..1b08e3324d7 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -15,6 +15,7 @@ module Backup
     # Copy files from public/files to backup/files
     def dump
       FileUtils.mkdir_p(Gitlab.config.backup.path)
+      FileUtils.rm_f(backup_tarball)
       run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600])
     end
 
-- 
cgit v1.2.1


From e789644783fae55f1095ffcc38b32f810f549caa Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 6 Oct 2015 15:21:15 +0200
Subject: Keep old path: db/database.sql.gz

Documentation elsewhere refers to this internal path, let's keep
it.
---
 lib/backup/database.rb | 3 ++-
 lib/backup/manager.rb  | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 4bdf6e1c628..fe0434361e8 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -6,10 +6,11 @@ module Backup
 
     def initialize
       @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
-      @db_file_name = File.join(Gitlab.config.backup.path, 'database.sql.gz')
+      @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
     end
 
     def dump
+      FileUtils.mkdir_p(File.dirname(db_file_name))
       FileUtils.rm_f(db_file_name)
       compress_rd, compress_wr = IO.pipe
       compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600])
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 53e79d4d1f7..f011fd03de0 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,11 +150,11 @@ module Backup
     private
 
     def backup_contents
-      folders_to_backup + ["database.sql.gz", "uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
     end
 
     def folders_to_backup
-      folders = %w{repositories}
+      folders = %w{repositories db}
 
       if ENV["SKIP"]
         return folders.reject{ |folder| ENV["SKIP"].include?(folder) }
-- 
cgit v1.2.1


From 7d58489fd908b2263f02e8919b1bd0b3fae1201d Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 6 Oct 2015 15:22:08 +0200
Subject: Remove unused variable

---
 lib/backup/files.rb | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 1b08e3324d7..5a210a0e464 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -2,14 +2,13 @@ require 'open3'
 
 module Backup
   class Files
-    attr_reader :name, :app_files_dir, :backup_tarball, :backup_dir, :files_parent_dir
+    attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir
 
     def initialize(app_files_dir)
       @app_files_dir = File.realpath(app_files_dir)
       @name = File.basename(app_files_dir)
       @files_parent_dir = File.realpath(File.join(@app_files_dir, '..'))
-      @backup_dir = Gitlab.config.backup.path
-      @backup_tarball = File.join(@backup_dir, name + '.tar.gz')
+      @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz')
     end
 
     # Copy files from public/files to backup/files
-- 
cgit v1.2.1


From 901f5445785a754227d8b77ca535947ab8cbbfca Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Tue, 6 Oct 2015 15:38:21 +0200
Subject: Remove superfluous mkdir -p

---
 lib/backup/database.rb | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib')

diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index fe0434361e8..67b2a64bd10 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -16,8 +16,6 @@ module Backup
       compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600])
       compress_rd.close
 
-      FileUtils.mkdir_p(Gitlab.config.backup.path)
-
       dump_pid = case config["adapter"]
       when /^mysql/ then
         $progress.print "Dumping MySQL database #{config['database']} ... "
-- 
cgit v1.2.1


From 58260a0327a953499a07e9cad8d9aaad2d25699b Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Fri, 16 Oct 2015 17:16:17 +0200
Subject: Do no rely on basename of builds, uploads

---
 lib/backup/builds.rb  | 6 +++++-
 lib/backup/files.rb   | 9 +++++----
 lib/backup/uploads.rb | 6 +++++-
 3 files changed, 15 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
index d269f8e260c..800f30c2144 100644
--- a/lib/backup/builds.rb
+++ b/lib/backup/builds.rb
@@ -1,7 +1,11 @@
 module Backup
   class Builds < Files
     def initialize
-      super(Settings.gitlab_ci.builds_path)
+      super('builds', Settings.gitlab_ci.builds_path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
     end
   end
 end
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 5a210a0e464..654b4d1c896 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -4,9 +4,9 @@ module Backup
   class Files
     attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir
 
-    def initialize(app_files_dir)
+    def initialize(name, app_files_dir)
+      @name = name
       @app_files_dir = File.realpath(app_files_dir)
-      @name = File.basename(app_files_dir)
       @files_parent_dir = File.realpath(File.join(@app_files_dir, '..'))
       @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz')
     end
@@ -15,13 +15,14 @@ module Backup
     def dump
       FileUtils.mkdir_p(Gitlab.config.backup.path)
       FileUtils.rm_f(backup_tarball)
-      run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+      run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600])
     end
 
     def restore
       backup_existing_files_dir
+      create_files_dir
 
-      run_pipeline!([%W(gzip -cd), %W(tar -C #{files_parent_dir} -xf -)], in: backup_tarball)
+      run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball)
     end
 
     def backup_existing_files_dir
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index 7c0838cc8b7..0a0ec564ba4 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -2,7 +2,11 @@ module Backup
   class Uploads < Files
 
     def initialize
-      super(Rails.root.join('public/uploads'))
+      super('uploads', Rails.root.join('public/uploads'))
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir)
     end
   end
 end
-- 
cgit v1.2.1


From 5a5069969ce8e9184e36abbb7623bf474d5869e9 Mon Sep 17 00:00:00 2001
From: Jonathan Schoeffling <jonathan.schoeffling@lmco.com>
Date: Sun, 14 Jun 2015 18:04:20 -0400
Subject: Add support for searching commit log messages

Include the log messages of recent commits in project-level search
results, providing functionality similar to 'git log --grep'.

Update repository model rspec tests to validate the output of
Repository#commits_with_log_matching.
---
 lib/gitlab/project_search_results.rb | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 0a2be605af9..3bf98699bcb 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -20,6 +20,8 @@ module Gitlab
         Kaminari.paginate_array(blobs).page(page).per(per_page)
       when 'wiki_blobs'
         Kaminari.paginate_array(wiki_blobs).page(page).per(per_page)
+      when 'commits'
+        Kaminari.paginate_array(commits).page(page).per(per_page)
       else
         super
       end
@@ -27,7 +29,7 @@ module Gitlab
 
     def total_count
       @total_count ||= issues_count + merge_requests_count + blobs_count +
-                       notes_count + wiki_blobs_count
+                       notes_count + wiki_blobs_count + commits_count
     end
 
     def blobs_count
@@ -42,6 +44,10 @@ module Gitlab
       @wiki_blobs_count ||= wiki_blobs.count
     end
 
+    def commits_count
+      @commits_count ||= commits.count
+    end
+
     private
 
     def blobs
@@ -70,6 +76,10 @@ module Gitlab
       Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC')
     end
 
+    def commits
+      project.repository.commits_with_log_matching(query)
+    end
+
     def limit_project_ids
       [project.id]
     end
-- 
cgit v1.2.1


From afdb53baecb2ef6d57f8ed957213260825c67805 Mon Sep 17 00:00:00 2001
From: Mike Chmielewski <code@mikec.123mail.org>
Date: Wed, 21 Oct 2015 01:04:19 +0000
Subject: Added @commits to list of tags.

---
 lib/tasks/spinach.rake | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
index c8881be0954..d5a96fd38f4 100644
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -5,7 +5,7 @@ namespace :spinach do
   task :project do
     cmds = [
       %W(rake gitlab:setup),
-      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
+      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits),
     ]
     run_commands(cmds)
   end
@@ -14,7 +14,7 @@ namespace :spinach do
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
+      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits),
     ]
     run_commands(cmds)
   end
@@ -33,4 +33,4 @@ def run_commands(cmds)
   cmds.each do |cmd|
     system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
   end
-end
+end
\ No newline at end of file
-- 
cgit v1.2.1


From b1f4aaa5e753e6e7cdefd84226839123df59b382 Mon Sep 17 00:00:00 2001
From: Michael Chmielewski <code@mikec.123mail.org>
Date: Tue, 27 Oct 2015 21:16:56 -0400
Subject: Trying to incorporate suggestions from comments on Merge Request 1661

---
 lib/gitlab/project_search_results.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 3bf98699bcb..54389f7d662 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -77,7 +77,7 @@ module Gitlab
     end
 
     def commits
-      project.repository.commits_with_log_matching(query)
+      project.repository.find_commits_with_matching_log(query)
     end
 
     def limit_project_ids
-- 
cgit v1.2.1


From c5132e94e1c44884f8f9b399c4def20154b6c0ce Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Thu, 29 Oct 2015 14:21:24 +0100
Subject: Switch to gitlab-workhorse

---
 lib/gitlab/backend/grack_auth.rb          |  2 +-
 lib/support/init.d/gitlab                 | 68 +++++++++++++++----------------
 lib/support/init.d/gitlab.default.example | 11 +++--
 lib/support/nginx/gitlab                  | 22 +++++-----
 lib/support/nginx/gitlab-ssl              | 22 +++++-----
 5 files changed, 61 insertions(+), 64 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 85a2d1a93a7..440ef5a3cb3 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -34,7 +34,7 @@ module Grack
       auth!
 
       if project && authorized_request?
-        # Tell gitlab-git-http-server the request is OK, and what the GL_ID is
+        # Tell gitlab-workhorse the request is OK, and what the GL_ID is
         render_grack_auth_ok
       elsif @user.nil? && !@ci
         unauthorized
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index a80e7e77430..f0a6c2b30e9 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -37,10 +37,9 @@ web_server_pid_path="$pid_path/unicorn.pid"
 sidekiq_pid_path="$pid_path/sidekiq.pid"
 mail_room_enabled=false
 mail_room_pid_path="$pid_path/mail_room.pid"
-gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid"
-gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080"
-gitlab_git_http_server_repo_root='/home/git/repositories'
-gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log"
+gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid"
+gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080"
+gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log"
 shell_path="/bin/bash"
 
 # Read configuration variable file if it is present
@@ -76,8 +75,8 @@ check_pids(){
   else
     spid=0
   fi
-  if [ -f "$gitlab_git_http_server_pid_path" ]; then
-    hpid=$(cat "$gitlab_git_http_server_pid_path")
+  if [ -f "$gitlab_workhorse_pid_path" ]; then
+    hpid=$(cat "$gitlab_workhorse_pid_path")
   else
     hpid=0
   fi
@@ -94,7 +93,7 @@ check_pids(){
 wait_for_pids(){
   # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid
   i=0;
-  while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_git_http_server_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do
+  while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do
     sleep 0.1;
     i=$((i+1))
     if [ $((i%10)) = 0 ]; then
@@ -131,9 +130,9 @@ check_status(){
   fi
   if [ $hpid -ne 0 ]; then
     kill -0 "$hpid" 2>/dev/null
-    gitlab_git_http_server_status="$?"
+    gitlab_workhorse_status="$?"
   else
-    gitlab_git_http_server_status="-1"
+    gitlab_workhorse_status="-1"
   fi
   if [ "$mail_room_enabled" = true ]; then
     if [ $mpid -ne 0 ]; then
@@ -143,7 +142,7 @@ check_status(){
       mail_room_status="-1"
     fi
   fi
-  if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_git_http_server_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then
+  if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then
     gitlab_status=0
   else
     # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html
@@ -171,9 +170,9 @@ check_stale_pids(){
       exit 1
     fi
   fi
-  if [ "$hpid" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ]; then
-    echo "Removing stale gitlab-git-http-server pid. This is most likely caused by gitlab-git-http-server crashing the last time it ran."
-    if ! rm "$gitlab_git_http_server_pid_path"; then
+  if [ "$hpid" != "0" ] && [ "$gitlab_workhorse_status" != "0" ]; then
+    echo "Removing stale gitlab-workhorse pid. This is most likely caused by gitlab-workhorse crashing the last time it ran."
+    if ! rm "$gitlab_workhorse_pid_path"; then
       echo "Unable to remove stale pid, exiting"
       exit 1
     fi
@@ -190,7 +189,7 @@ check_stale_pids(){
 ## If no parts of the service is running, bail out.
 exit_if_not_running(){
   check_stale_pids
-  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
+  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
     echo "GitLab is not running."
     exit
   fi
@@ -206,8 +205,8 @@ start_gitlab() {
   if [ "$sidekiq_status" != "0" ]; then
     echo "Starting GitLab Sidekiq"
   fi
-  if [ "$gitlab_git_http_server_status" != "0" ]; then
-    echo "Starting gitlab-git-http-server"
+  if [ "$gitlab_workhorse_status" != "0" ]; then
+    echo "Starting gitlab-workhorse"
   fi
   if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then
     echo "Starting GitLab MailRoom"
@@ -230,15 +229,14 @@ start_gitlab() {
     RAILS_ENV=$RAILS_ENV bin/background_jobs start &
   fi
 
-  if [ "$gitlab_git_http_server_status" = "0" ]; then
-    echo "The gitlab-git-http-server is already running with pid $spid, not restarting"
+  if [ "$gitlab_workhorse_status" = "0" ]; then
+    echo "The gitlab-workhorse is already running with pid $spid, not restarting"
   else
-    # No need to remove a socket, gitlab-git-http-server does this itself
-    $app_root/bin/daemon_with_pidfile $gitlab_git_http_server_pid_path  \
-      $app_root/../gitlab-git-http-server/gitlab-git-http-server \
-        $gitlab_git_http_server_options \
-        $gitlab_git_http_server_repo_root \
-      >> $gitlab_git_http_server_log 2>&1 &
+    # No need to remove a socket, gitlab-workhorse does this itself
+    $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path  \
+      $app_root/../gitlab-workhorse/gitlab-workhorse \
+        $gitlab_workhorse_options \
+      >> $gitlab_workhorse_log 2>&1 &
   fi
 
   if [ "$mail_room_enabled" = true ]; then
@@ -268,9 +266,9 @@ stop_gitlab() {
     echo "Shutting down GitLab Sidekiq"
     RAILS_ENV=$RAILS_ENV bin/background_jobs stop
   fi
-  if [ "$gitlab_git_http_server_status" = "0" ]; then
-    echo "Shutting down gitlab-git-http-server"
-    kill -- $(cat $gitlab_git_http_server_pid_path)
+  if [ "$gitlab_workhorse_status" = "0" ]; then
+    echo "Shutting down gitlab-workhorse"
+    kill -- $(cat $gitlab_workhorse_pid_path)
   fi
   if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then
     echo "Shutting down GitLab MailRoom"
@@ -278,11 +276,11 @@ stop_gitlab() {
   fi
 
   # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script.
-  while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do
+  while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do
     sleep 1
     check_status
     printf "."
-    if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
+    if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
       printf "\n"
       break
     fi
@@ -292,7 +290,7 @@ stop_gitlab() {
   # Cleaning up unused pids
   rm "$web_server_pid_path" 2>/dev/null
   # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up it's own pid.
-  rm -f "$gitlab_git_http_server_pid_path"
+  rm -f "$gitlab_workhorse_pid_path"
   if [ "$mail_room_enabled" = true ]; then
     rm "$mail_room_pid_path" 2>/dev/null
   fi
@@ -303,7 +301,7 @@ stop_gitlab() {
 ## Prints the status of GitLab and it's components.
 print_status() {
   check_status
-  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
+  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
     echo "GitLab is not running."
     return
   fi
@@ -317,10 +315,10 @@ print_status() {
   else
       printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n"
   fi
-  if [ "$gitlab_git_http_server_status" = "0" ]; then
-      echo "The gitlab-git-http-server with pid $hpid is running."
+  if [ "$gitlab_workhorse_status" = "0" ]; then
+      echo "The gitlab-workhorse with pid $hpid is running."
   else
-      printf "The gitlab-git-http-server is \033[31mnot running\033[0m.\n"
+      printf "The gitlab-workhorse is \033[31mnot running\033[0m.\n"
   fi
   if [ "$mail_room_enabled" = true ]; then
     if [ "$mail_room_status" = "0" ]; then
@@ -360,7 +358,7 @@ reload_gitlab(){
 ## Restarts Sidekiq and Unicorn.
 restart_gitlab(){
   check_status
-  if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then
+  if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then
     stop_gitlab
   fi
   start_gitlab
diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example
index aab5acaa72c..79ae8e0ae55 100755
--- a/lib/support/init.d/gitlab.default.example
+++ b/lib/support/init.d/gitlab.default.example
@@ -30,15 +30,14 @@ web_server_pid_path="$pid_path/unicorn.pid"
 # The default is "$pid_path/sidekiq.pid"
 sidekiq_pid_path="$pid_path/sidekiq.pid"
 
-gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid"
-# The -listenXxx settings determine where gitlab-git-http-server
+gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid"
+# The -listenXxx settings determine where gitlab-workhorse
 # listens for connections from NGINX. To listen on localhost:8181, write
 # '-listenNetwork tcp -listenAddr localhost:8181'.
-# The -authBackend setting tells gitlab-git-http-server where it can reach
+# The -authBackend setting tells gitlab-workhorse where it can reach
 # Unicorn.
-gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080"
-gitlab_git_http_server_repo_root="/home/git/repositories"
-gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log"
+gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080"
+gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log"
 
 # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled.
 # This is required for the Reply by email feature.
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index 1e55c5a0486..e767027dc29 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -38,8 +38,8 @@ upstream gitlab {
   server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0;
 }
 
-upstream gitlab-git-http-server {
-  server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0;
+upstream gitlab-workhorse {
+  server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
 }
 
 ## Normal HTTP host
@@ -114,24 +114,24 @@ server {
   }
 
   location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
-    # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-    error_page 418 = @gitlab-git-http-server;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
     return 418;
   }
 
   location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive {
-    # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-    error_page 418 = @gitlab-git-http-server;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
     return 418;
   }
 
   location ~ ^/api/v3/projects/.*/repository/archive {
-    # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-    error_page 418 = @gitlab-git-http-server;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
     return 418;
   }
 
-  location @gitlab-git-http-server {
+  location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
     # gzip off;
@@ -147,7 +147,7 @@ server {
 
     # The following settings only work with NGINX 1.7.11 or newer
     #
-    # # Pass chunked request bodies to gitlab-git-http-server as-is
+    # # Pass chunked request bodies to gitlab-workhorse as-is
     # proxy_request_buffering off;
     # proxy_http_version 1.1;
 
@@ -156,7 +156,7 @@ server {
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     proxy_set_header    X-Forwarded-Proto   $scheme;
 
-    proxy_pass http://gitlab-git-http-server;
+    proxy_pass http://gitlab-workhorse;
   }
 
   ## Enable gzip compression as per rails guide:
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 08641bbcc17..4d31e31f8d5 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -42,8 +42,8 @@ upstream gitlab {
   server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0;
 }
 
-upstream gitlab-git-http-server {
-  server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0;
+upstream gitlab-workhorse {
+  server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
 }
 
 ## Redirects all HTTP traffic to the HTTPS host
@@ -161,24 +161,24 @@ server {
   }
 
   location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
-    # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-    error_page 418 = @gitlab-git-http-server;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
     return 418;
   }
 
   location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive {
-    # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-    error_page 418 = @gitlab-git-http-server;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
     return 418;
   }
 
   location ~ ^/api/v3/projects/.*/repository/archive {
-    # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-    error_page 418 = @gitlab-git-http-server;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
     return 418;
   }
 
-  location @gitlab-git-http-server {
+  location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
     gzip off;
@@ -194,7 +194,7 @@ server {
 
     # The following settings only work with NGINX 1.7.11 or newer
     #
-    # # Pass chunked request bodies to gitlab-git-http-server as-is
+    # # Pass chunked request bodies to gitlab-workhorse as-is
     # proxy_request_buffering off;
     # proxy_http_version 1.1;
 
@@ -203,7 +203,7 @@ server {
     proxy_set_header    X-Forwarded-Ssl     on;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     proxy_set_header    X-Forwarded-Proto   $scheme;
-    proxy_pass http://gitlab-git-http-server;
+    proxy_pass http://gitlab-workhorse;
   }
 
   ## Enable gzip compression as per rails guide:
-- 
cgit v1.2.1


From 3be9d2c422b8651498abec3a2ee9bb6a3685f040 Mon Sep 17 00:00:00 2001
From: Ben Ford <ben.ford@puppetlabs.com>
Date: Mon, 19 Oct 2015 14:52:46 -0700
Subject: Add ability to create directories in the editor

Simply type a name with a `/` directory separator and new directories
will be created. This does not do the fancy UI work that github.com
does, but it will get the job done.

I could not find tests for file creation, so I didn't add a test for
this slight behaviour modification. I did test directory traversals
though, using both absolute paths like `/tmp/foo.txt` and relative paths
like `../../foo.txt`. Neither case escaped the repository, though
attempting to traverse with a relative path resulted in a 500 error that
did not affect application stability upon reload.
---
 lib/gitlab/regex.rb | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 9f1adc860d1..53ab2686b43 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -51,6 +51,23 @@ module Gitlab
       "can contain only letters, digits, '_', '-' and '.'. "
     end
 
+    def file_path_regex
+      @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze
+    end
+
+    def file_path_regex_message
+      "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. "
+    end
+
+
+    def directory_traversal_regex
+      @directory_traversal_regex ||= /\.{2}/.freeze
+    end
+
+    def directory_traversal_regex_message
+      "cannot include directory traversal. "
+    end
+
 
     def archive_formats_regex
       #                           |zip|tar|    tar.gz    |         tar.bz2         |
-- 
cgit v1.2.1


From c03da1caade637298ca96e59cea990b0827539c9 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 2 Nov 2015 14:44:06 +0100
Subject: Extend yml syntax for only and except to support specifying
 repository path

---
 lib/ci/gitlab_ci_yaml_processor.rb | 64 ++++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 30 deletions(-)

(limited to 'lib')

diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index efcd2faffc7..0f57a4f53ab 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -7,10 +7,11 @@ module Ci
     ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
     ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
 
-    attr_reader :before_script, :image, :services, :variables
+    attr_reader :before_script, :image, :services, :variables, :path
 
-    def initialize(config)
+    def initialize(config, path = nil)
       @config = YAML.load(config)
+      @path = path
 
       unless @config.is_a? Hash
         raise ValidationError, "YAML should be a hash"
@@ -63,26 +64,6 @@ module Ci
       end
     end
 
-    def process?(only_params, except_params, ref, tag)
-      return true if only_params.nil? && except_params.nil?
-
-      if only_params
-        return true if tag && only_params.include?("tags")
-        return true if !tag && only_params.include?("branches")
-        
-        only_params.find do |pattern|
-          match_ref?(pattern, ref)
-        end
-      else
-        return false if tag && except_params.include?("tags")
-        return false if !tag && except_params.include?("branches")
-
-        except_params.each do |pattern|
-          return false if match_ref?(pattern, ref)
-        end
-      end
-    end
-
     def build_job(name, job)
       {
         stage_idx: stages.index(job[:stage]),
@@ -101,14 +82,6 @@ module Ci
       }
     end
 
-    def match_ref?(pattern, ref)
-      if pattern.first == "/" && pattern.last == "/"
-        Regexp.new(pattern[1...-1]) =~ ref
-      else
-        pattern == ref
-      end
-    end
-
     def normalize_script(script)
       if script.is_a? Array
         script.join("\n")
@@ -208,5 +181,36 @@ module Ci
     def validate_string(value)
       value.is_a?(String) || value.is_a?(Symbol)
     end
+
+    def process?(only_params, except_params, ref, tag)
+      if only_params.present?
+        return false unless matching?(only_params, ref, tag)
+      end
+
+      if except_params.present?
+        return false if matching?(except_params, ref, tag)
+      end
+
+      true
+    end
+
+    def matching?(patterns, ref, tag)
+      patterns.any? do |pattern|
+        match_ref?(pattern, ref, tag)
+      end
+    end
+
+    def match_ref?(pattern, ref, tag)
+      pattern, path = pattern.split('@', 2)
+      return false if path && path != self.path
+      return true if tag && pattern == 'tags'
+      return true if !tag && pattern == 'branches'
+
+      if pattern.first == "/" && pattern.last == "/"
+        Regexp.new(pattern[1...-1]) =~ ref
+      else
+        pattern == ref
+      end
+    end
   end
 end
-- 
cgit v1.2.1


From e032a7288f4103a6d5980da6af4040bc9271bd99 Mon Sep 17 00:00:00 2001
From: Jacob Vosmaer <contact@jacobvosmaer.nl>
Date: Mon, 2 Nov 2015 16:02:47 +0100
Subject: Spread out runner contacted_at updates

---
 lib/ci/api/helpers.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index e602cda81d6..7e4986b6af3 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -16,7 +16,9 @@ module Ci
       end
 
       def update_runner_last_contact
-        if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY
+        # Use a random threshold to prevent beating DB updates
+        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
+        if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age
           current_runner.update_attributes(contacted_at: Time.now)
         end
       end
-- 
cgit v1.2.1


From 1b8d324762787805fabee72d585ac40e244af4d7 Mon Sep 17 00:00:00 2001
From: Valery Sizov <vsv2711@gmail.com>
Date: Mon, 2 Nov 2015 14:14:38 +0200
Subject: Add ability to fetch the commit ID of the last commit that actually
 touched a file

---
 lib/api/files.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/files.rb b/lib/api/files.rb
index 308c84dd135..a7a768f8895 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -43,7 +43,8 @@ module API
       #   "content": "IyA9PSBTY2hlbWEgSW5mb3...",
       #   "ref": "master",
       #   "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83",
-      #   "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50"
+      #   "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50",
+      #   "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d",
       # }
       #
       get ":id/repository/files" do
@@ -71,6 +72,7 @@ module API
             ref: ref,
             blob_id: blob.id,
             commit_id: commit.id,
+            last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id
           }
         else
           not_found! 'File'
-- 
cgit v1.2.1


From 810c91fe35db6a83c9c517e03d07dc1795922646 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Mon, 2 Nov 2015 16:39:24 +0100
Subject: Refactor search by commits message

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/project_search_results.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 54389f7d662..c873ef3771e 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -77,7 +77,7 @@ module Gitlab
     end
 
     def commits
-      project.repository.find_commits_with_matching_log(query)
+      project.repository.find_commits_by_message(query)
     end
 
     def limit_project_ids
-- 
cgit v1.2.1


From 60c5b52ee906639e8bc0408a4a63d50862580fc8 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Mon, 2 Nov 2015 15:04:43 -0500
Subject: Explicitly require backup/files

---
 lib/backup/builds.rb  | 2 ++
 lib/backup/uploads.rb | 2 ++
 2 files changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
index 800f30c2144..635967f4bd4 100644
--- a/lib/backup/builds.rb
+++ b/lib/backup/builds.rb
@@ -1,3 +1,5 @@
+require 'backup/files'
+
 module Backup
   class Builds < Files
     def initialize
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index 0a0ec564ba4..9261f77f3c9 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -1,3 +1,5 @@
+require 'backup/files'
+
 module Backup
   class Uploads < Files
 
-- 
cgit v1.2.1


From 28f6fba97cb9753c43d109ef4f43439413d4eb69 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Tue, 3 Nov 2015 10:16:09 +0100
Subject: Fix commits search for empty repository

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/project_search_results.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index c873ef3771e..70de6a74e76 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -77,7 +77,11 @@ module Gitlab
     end
 
     def commits
-      project.repository.find_commits_by_message(query)
+      if project.empty_repo? || query.blank?
+        []
+      else
+        project.repository.find_commits_by_message(query).compact
+      end
     end
 
     def limit_project_ids
-- 
cgit v1.2.1


From 9479496f7584336788fc61121c096dfbcb3f6e4a Mon Sep 17 00:00:00 2001
From: Valery Sizov <valery@gitlab.com>
Date: Mon, 2 Nov 2015 16:56:44 +0200
Subject: Add added, modified and removed properties to commit object in
 webhook

---
 lib/gitlab/push_data_builder.rb | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb
index d010ade704e..fa068d50763 100644
--- a/lib/gitlab/push_data_builder.rb
+++ b/lib/gitlab/push_data_builder.rb
@@ -18,7 +18,10 @@ module Gitlab
       #     homepage: String,
       #   },
       #   commits: Array,
-      #   total_commits_count: Fixnum
+      #   total_commits_count: Fixnum,
+      #   added: ["CHANGELOG"],
+      #   modified: [],
+      #   removed: ["tmp/file.txt"]
       # }
       #
       def build(project, user, oldrev, newrev, ref, commits = [], message = nil)
@@ -33,6 +36,8 @@ module Gitlab
         commit_attrs = commits_limited.map(&:hook_attrs)
 
         type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push"
+
+        repo_changes = repo_changes(project, newrev, oldrev)
         # Hash to be passed as post_receive_data
         data = {
           object_kind: type,
@@ -55,7 +60,10 @@ module Gitlab
             visibility_level: project.visibility_level
           },
           commits: commit_attrs,
-          total_commits_count: commits_count
+          total_commits_count: commits_count,
+          added: repo_changes[:added],
+          modified: repo_changes[:modified],
+          removed: repo_changes[:removed]
         }
 
         data
@@ -86,6 +94,27 @@ module Gitlab
           newrev
         end
       end
+
+      def repo_changes(project, newrev, oldrev)
+        changes = { added: [], modified: [], removed: [] }
+        compare_result = CompareService.new.
+          execute(project, newrev, project, oldrev)
+
+        if compare_result
+          compare_result.diffs.each do |diff|
+            case true
+            when diff.deleted_file
+              changes[:removed] << diff.old_path
+            when diff.renamed_file, diff.new_file
+              changes[:added] << diff.new_path
+            else
+              changes[:modified] << diff.new_path
+            end
+          end
+        end
+
+        changes
+      end
     end
   end
 end
-- 
cgit v1.2.1


From 98dcad2762930b0dc967ff8e67f9f795228d2342 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Tue, 3 Nov 2015 17:58:26 +0100
Subject: Use proper labels for OAuth providers

---
 lib/gitlab/o_auth/provider.rb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb
index 90c3fe8da33..9ad7a38d505 100644
--- a/lib/gitlab/o_auth/provider.rb
+++ b/lib/gitlab/o_auth/provider.rb
@@ -1,6 +1,12 @@
 module Gitlab
   module OAuth
     class Provider
+      LABELS = {
+        "github"         => "GitHub",
+        "gitlab"         => "GitLab.com",
+        "google_oauth2"  => "Google"
+      }.freeze
+
       def self.providers
         Devise.omniauth_providers
       end
@@ -23,8 +29,9 @@ module Gitlab
       end
 
       def self.label_for(name)
+        name = name.to_s
         config = config_for(name)
-        (config && config['label']) || name.to_s.titleize
+        (config && config['label']) || LABELS[name] || name.titleize
       end
     end
   end
-- 
cgit v1.2.1


From d09d62b6b875102b7a334fcf9e689537e1f25013 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Tue, 3 Nov 2015 17:10:38 -0500
Subject: Replace all usages of `git` command with configurable binary path

Closes #3311
---
 lib/backup/repository.rb        |  6 +++---
 lib/gitlab/force_push_check.rb  |  2 +-
 lib/gitlab/git_ref_validator.rb |  2 +-
 lib/gitlab/upgrader.rb          |  8 ++++----
 lib/tasks/gitlab/check.rake     |  2 +-
 lib/tasks/gitlab/shell.rake     | 10 +++++-----
 6 files changed, 15 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 4d70f7883dd..a82a7e1f7bf 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -35,7 +35,7 @@ module Backup
           if wiki.repository.empty?
             $progress.puts " [SKIPPED]".cyan
           else
-            cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
+            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
             output, status = Gitlab::Popen.popen(cmd)
             if status.zero?
               $progress.puts " [DONE]".green
@@ -67,7 +67,7 @@ module Backup
           FileUtils.mkdir_p(path_to_repo(project))
           cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)})
         else
-          cmd = %W(git init --bare #{path_to_repo(project)})
+          cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)})
         end
 
         if system(*cmd, silent)
@@ -87,7 +87,7 @@ module Backup
           # that was initialized with ProjectWiki.new() and then
           # try to restore with 'git clone --bare'.
           FileUtils.rm_rf(path_to_repo(wiki))
-          cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
+          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
 
           if system(*cmd, silent)
             $progress.puts " [DONE]".green
diff --git a/lib/gitlab/force_push_check.rb b/lib/gitlab/force_push_check.rb
index fdb6a35c78d..93c6a5bb7f5 100644
--- a/lib/gitlab/force_push_check.rb
+++ b/lib/gitlab/force_push_check.rb
@@ -7,7 +7,7 @@ module Gitlab
       if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
         false
       else
-        missed_refs, _ = Gitlab::Popen.popen(%W(git --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev}))
+        missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev}))
         missed_refs.split("\n").size > 0
       end
     end
diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb
index 39d17def930..4d83d8e72a8 100644
--- a/lib/gitlab/git_ref_validator.rb
+++ b/lib/gitlab/git_ref_validator.rb
@@ -6,7 +6,7 @@ module Gitlab
     # Returns true for a valid reference name, false otherwise
     def validate(ref_name)
       Gitlab::Utils.system_silent(
-        %W(git check-ref-format refs/#{ref_name}))
+        %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name}))
     end
   end
 end
diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb
index cf040971c6e..f3567f3ef85 100644
--- a/lib/gitlab/upgrader.rb
+++ b/lib/gitlab/upgrader.rb
@@ -50,15 +50,15 @@ module Gitlab
     end
 
     def fetch_git_tags
-      remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git))
+      remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git))
       remote_tags.split("\n").grep(/tags\/v#{current_version.major}/)
     end
 
     def update_commands
       {
-        "Stash changed files" => %W(git stash),
-        "Get latest code" => %W(git fetch),
-        "Switch to new version" => %W(git checkout v#{latest_version}),
+        "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash),
+        "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch),
+        "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}),
         "Install gems" => %W(bundle),
         "Migrate DB" => %W(bundle exec rake db:migrate),
         "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile),
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 2e73f792a9d..a25fac62cfc 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -824,7 +824,7 @@ namespace :gitlab do
         repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
         repo_dirs.each do |dir|
           puts "\nChecking repo at #{dir}"
-          system(*%w(git fsck), chdir: dir)
+          system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir)
         end
       end
     end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 3c0cc763d17..dd61632e557 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -17,7 +17,7 @@ namespace :gitlab do
 
       # Clone if needed
       unless File.directory?(target_dir)
-        system(*%W(git clone -- #{args.repo} #{target_dir}))
+        system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir}))
       end
 
       # Make sure we're on the right tag
@@ -27,7 +27,7 @@ namespace :gitlab do
         reseted = reset_to_commit(args)
 
         unless reseted
-          system(*%W(git fetch origin))
+          system(*%W(#{Gitlab.config.git.bin_path} fetch origin))
           reset_to_commit(args)
         end
 
@@ -128,14 +128,14 @@ namespace :gitlab do
   end
 
   def reset_to_commit(args)
-    tag, status = Gitlab::Popen.popen(%W(git describe -- #{args.tag}))
+    tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag}))
 
     unless status.zero?
-      tag, status = Gitlab::Popen.popen(%W(git describe -- origin/#{args.tag}))
+      tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag}))
     end
 
     tag = tag.strip
-    system(*%W(git reset --hard #{tag}))
+    system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
   end
 end
 
-- 
cgit v1.2.1


From 91cbf9db0c5c95381e7d422fd684e623d58fadab Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Mon, 26 Oct 2015 07:26:03 +0100
Subject: Add allow_failure field to commit status API

Closes #3196
---
 lib/api/entities.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 883a5e14b17..20cadae2291 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -231,7 +231,7 @@ module API
 
     class CommitStatus < Grape::Entity
       expose :id, :sha, :ref, :status, :name, :target_url, :description,
-             :created_at, :started_at, :finished_at
+             :created_at, :started_at, :finished_at, :allow_failure
       expose :author, using: Entities::UserBasic
     end
 
-- 
cgit v1.2.1


From b18671a1b2c565a87663544441000063f6b83c8e Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 3 Nov 2015 14:45:41 +0100
Subject: Enable shared runners for all new projects

---
 lib/gitlab/current_settings.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 0ea1b6a2f6f..cd84afa31d5 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -23,7 +23,8 @@ module Gitlab
         restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],
         max_attachment_size: Settings.gitlab['max_attachment_size'],
         session_expire_delay: Settings.gitlab['session_expire_delay'],
-        import_sources: Settings.gitlab['import_sources']
+        import_sources: Settings.gitlab['import_sources'],
+        shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
       )
     end
 
-- 
cgit v1.2.1


From dec3e4ce64df5f71a7cba7734cada1baa79242cd Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Wed, 4 Nov 2015 19:13:19 +0100
Subject: Added Sherlock, a custom profiling tool for GitLab

Sherlock will be a new GitLab specific tool for measuring the
performance of Rails requests (and SideKiq jobs at some point). Some of
the things that are currently tracked:

* SQL queries along with their timings, backtraces and query plans
  (using "EXPLAIN ANALYZE" for PostgreSQL and regular "EXPLAIN" for
  MySQL)
* Timings of application files (including views) on a per line basis
* Some meta data such as the request method, path, total duration, etc

More tracking (e.g. Rugged or gitlab-shell timings) might be added in
the future.

Sherlock will replace any existing tools we have used so far (e.g.
active_record_query_trace and rack-mini-profiler), hence the
corresponding Gems have been removed from the Gemfile.

Sherlock can be enabled by starting Rails as following:

    ENABLE_SHERLOCK=1 bundle exec rails s

Recorded transactions can be found at `/sherlock/transactions`.
---
 lib/gitlab/sherlock.rb               | 20 ++++++++
 lib/gitlab/sherlock/collection.rb    | 42 ++++++++++++++++
 lib/gitlab/sherlock/file_sample.rb   | 27 ++++++++++
 lib/gitlab/sherlock/line_profiler.rb | 60 ++++++++++++++++++++++
 lib/gitlab/sherlock/line_sample.rb   | 20 ++++++++
 lib/gitlab/sherlock/location.rb      | 22 +++++++++
 lib/gitlab/sherlock/middleware.rb    | 36 ++++++++++++++
 lib/gitlab/sherlock/query.rb         | 96 ++++++++++++++++++++++++++++++++++++
 lib/gitlab/sherlock/transaction.rb   | 85 +++++++++++++++++++++++++++++++
 9 files changed, 408 insertions(+)
 create mode 100644 lib/gitlab/sherlock.rb
 create mode 100644 lib/gitlab/sherlock/collection.rb
 create mode 100644 lib/gitlab/sherlock/file_sample.rb
 create mode 100644 lib/gitlab/sherlock/line_profiler.rb
 create mode 100644 lib/gitlab/sherlock/line_sample.rb
 create mode 100644 lib/gitlab/sherlock/location.rb
 create mode 100644 lib/gitlab/sherlock/middleware.rb
 create mode 100644 lib/gitlab/sherlock/query.rb
 create mode 100644 lib/gitlab/sherlock/transaction.rb

(limited to 'lib')

diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb
new file mode 100644
index 00000000000..c4b35b24ceb
--- /dev/null
+++ b/lib/gitlab/sherlock.rb
@@ -0,0 +1,20 @@
+require 'securerandom'
+require 'rblineprof' if RUBY_ENGINE == 'ruby'
+
+module Gitlab
+  module Sherlock
+    @collection = Collection.new
+
+    class << self
+      attr_reader :collection
+    end
+
+    def self.enabled?
+      Rails.env.development? && !!ENV['ENABLE_SHERLOCK']
+    end
+
+    def self.enable_line_profiler?
+      RUBY_ENGINE == 'ruby'
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb
new file mode 100644
index 00000000000..accdc6469bc
--- /dev/null
+++ b/lib/gitlab/sherlock/collection.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module Sherlock
+    class Collection
+      include Enumerable
+
+      def initialize
+        @transactions = []
+        @mutex = Mutex.new
+      end
+
+      def add(transaction)
+        synchronize { @transactions << transaction }
+      end
+
+      alias_method :<<, :add
+
+      def each(&block)
+        synchronize { @transactions.each(&block) }
+      end
+
+      def clear
+        synchronize { @transactions.clear }
+      end
+
+      def empty?
+        synchronize { @transactions.empty? }
+      end
+
+      def find_transaction(id)
+        find { |trans| trans.id == id }
+      end
+
+      def newest_first
+        sort { |a, b| b.finished_at <=> a.finished_at }
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb
new file mode 100644
index 00000000000..7a220de9abc
--- /dev/null
+++ b/lib/gitlab/sherlock/file_sample.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Sherlock
+    class FileSample
+      attr_reader :id, :file, :line_samples, :events, :duration
+
+      def initialize(file, line_samples, duration, events)
+        @id = SecureRandom.uuid
+        @file = file
+        @line_samples = line_samples
+        @duration = duration
+        @events = events
+      end
+
+      def relative_path
+        @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '')
+      end
+
+      def to_param
+        @id
+      end
+
+      def source
+        @source ||= File.read(@file)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
new file mode 100644
index 00000000000..a191b1e646d
--- /dev/null
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -0,0 +1,60 @@
+module Gitlab
+  module Sherlock
+    class LineProfiler
+      # The minimum amount of time that has to be spent in a file for it to be
+      # included in a list of samples.
+      MINIMUM_DURATION = 10.0
+
+      def profile(&block)
+        if RUBY_ENGINE == 'ruby'
+          profile_mri(&block)
+        else
+          raise NotImplementedError,
+            'Line profiling is not supported on this platform'
+        end
+      end
+
+      def profile_mri
+        retval  = nil
+        samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield }
+
+        file_samples = aggregate_rblineprof(samples)
+
+        [retval, file_samples]
+      end
+
+      # Returns an Array of file samples based on the output of rblineprof.
+      def aggregate_rblineprof(lineprof_stats)
+        samples = []
+
+        lineprof_stats.each do |(file, stats)|
+          source_lines = File.read(file).each_line.to_a
+          line_samples = []
+
+          total_duration = microsec_to_millisec(stats[0][0])
+          total_events   = stats[0][2]
+
+          next if total_duration <= MINIMUM_DURATION
+
+          stats[1..-1].each_with_index do |data, index|
+            next unless source_lines[index]
+
+            duration = microsec_to_millisec(data[0])
+            events   = data[2]
+
+            line_samples << LineSample.new(duration, events)
+          end
+
+          samples << FileSample.
+            new(file, line_samples, total_duration, total_events)
+        end
+
+        samples
+      end
+
+      def microsec_to_millisec(microsec)
+        microsec / 1000.0
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb
new file mode 100644
index 00000000000..38df7a88e4e
--- /dev/null
+++ b/lib/gitlab/sherlock/line_sample.rb
@@ -0,0 +1,20 @@
+module Gitlab
+  module Sherlock
+    class LineSample
+      attr_reader :duration, :events
+
+      def initialize(duration, events)
+        @duration = duration
+        @events = events
+      end
+
+      def percentage_of(total_duration)
+        (duration.to_f / total_duration) * 100.0
+      end
+
+      def majority_of?(total_duration)
+        percentage_of(total_duration) >= 30
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb
new file mode 100644
index 00000000000..8c0b77dce1a
--- /dev/null
+++ b/lib/gitlab/sherlock/location.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Sherlock
+    class Location
+      attr_reader :path, :line
+
+      SHERLOCK_DIR = File.dirname(__FILE__)
+
+      def self.from_ruby_location(location)
+        new(location.path, location.lineno)
+      end
+
+      def initialize(path, line)
+        @path = path
+        @line = line
+      end
+
+      def application?
+        @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb
new file mode 100644
index 00000000000..fca7be858eb
--- /dev/null
+++ b/lib/gitlab/sherlock/middleware.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Sherlock
+    # Rack middleware used for tracking request metrics.
+    class Middleware
+      CONTENT_TYPES = /text\/html|application\/json/i
+
+      IGNORE_PATHS = %r{^/sherlock}
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if instrument?(env)
+          call_with_instrumentation(env)
+        else
+          @app.call(env)
+        end
+      end
+
+      def call_with_instrumentation(env)
+        trans = Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI'])
+        retval = trans.run { @app.call(env) }
+
+        Sherlock.collection.add(trans)
+
+        retval
+      end
+
+      def instrument?(env)
+        !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES &&
+           env['REQUEST_URI'] !~ IGNORE_PATHS)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb
new file mode 100644
index 00000000000..af76e6fd2bf
--- /dev/null
+++ b/lib/gitlab/sherlock/query.rb
@@ -0,0 +1,96 @@
+module Gitlab
+  module Sherlock
+    class Query
+      attr_reader :id, :query, :started_at, :finished_at, :backtrace
+
+      PREFIX_NEWLINE = /
+        \s+(FROM
+          |(LEFT|RIGHT)?INNER\s+JOIN
+          |(LEFT|RIGHT)?OUTER\s+JOIN
+          |WHERE
+          |AND
+          |GROUP\s+BY
+          |ORDER\s+BY
+          |LIMIT
+          |OFFSET)\s+
+      /ix
+
+      def self.new_with_bindings(query, bindings, started_at, finished_at)
+        bindings.each_with_index do |(column, value), index|
+          quoted_value = ActiveRecord::Base.connection.quote(value)
+
+          query = query.gsub("$#{index + 1}", quoted_value)
+        end
+
+        new(query, started_at, finished_at)
+      end
+
+      def initialize(query, started_at, finished_at)
+        @id = SecureRandom.uuid
+        @query = query
+        @started_at = started_at
+        @finished_at = finished_at
+        @backtrace = caller_locations.map do |loc|
+          Location.from_ruby_location(loc)
+        end
+
+        unless @query.end_with?(';')
+          @query += ';'
+        end
+      end
+
+      def duration
+        @duration ||= (@finished_at - @started_at) * 1000.0
+      end
+
+      def to_param
+        @id
+      end
+
+      def formatted_query
+        @formatted_query ||= format_sql(@query)
+      end
+
+      def last_application_frame
+        @last_application_frame ||= @backtrace.find(&:application?)
+      end
+
+      def application_backtrace
+        @application_backtrace ||= @backtrace.select(&:application?)
+      end
+
+      def explain
+        unless @explain
+          ActiveRecord::Base.connection.transaction do
+            @explain = raw_explain(@query).values.flatten.join("\n")
+
+            # Roll back any queries that mutate data so we don't mess up
+            # anything when running explain on an INSERT, UPDATE, DELETE, etc.
+            raise ActiveRecord::Rollback
+          end
+        end
+
+        @explain
+      end
+
+      private
+
+      def raw_explain(query)
+        if Gitlab::Database.postgresql?
+          explain = "EXPLAIN ANALYZE #{query};"
+        else
+          explain = "EXPLAIN #{query};"
+        end
+
+        ActiveRecord::Base.connection.execute(explain)
+      end
+
+      def format_sql(query)
+        query.each_line.
+          map { |line| line.strip }.
+          join("\n").
+          gsub(PREFIX_NEWLINE) { "\n#{$1} " }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb
new file mode 100644
index 00000000000..5cb3e86aa4e
--- /dev/null
+++ b/lib/gitlab/sherlock/transaction.rb
@@ -0,0 +1,85 @@
+module Gitlab
+  module Sherlock
+    class Transaction
+      attr_reader :id, :type, :path, :queries, :file_samples, :started_at,
+        :finished_at
+
+      def initialize(type, path)
+        @id = SecureRandom.uuid
+        @type = type
+        @path = path
+        @duration = 0
+        @queries = []
+        @file_samples = []
+        @started_at = nil
+        @finished_at = nil
+        @thread = Thread.current
+      end
+
+      def run
+        @started_at = Time.now
+
+        subscriber = subscribe_to_active_record
+
+        retval = profile_lines { yield }
+
+        @finished_at = Time.now
+
+        ActiveSupport::Notifications.unsubscribe(subscriber)
+
+        retval
+      end
+
+      def duration
+        @started_at && @finished_at ? @finished_at - @started_at : 0
+      end
+
+      def to_param
+        @id
+      end
+
+      def sorted_queries
+        @queries.sort { |a, b| b.duration <=> a.duration }
+      end
+
+      def sorted_file_samples
+        @file_samples.sort { |a, b| b.duration <=> a.duration }
+      end
+
+      def find_query(id)
+        @queries.find { |query| query.id == id }
+      end
+
+      def find_file_sample(id)
+        @file_samples.find { |sample| sample.id == id }
+      end
+
+      def track_query(query, bindings, start, finish)
+        @queries << Query.new_with_bindings(query, bindings, start, finish)
+      end
+
+      def profile_lines
+        retval = nil
+
+        if Sherlock.enable_line_profiler?
+          retval, @file_samples = LineProfiler.new.profile { yield }
+        else
+          retval = yield
+        end
+
+        retval
+      end
+
+      def subscribe_to_active_record
+        ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
+          # In case somebody uses a multi-threaded server locally (e.g. Puma) we
+          # _only_ want to track queries that originate from the transaction
+          # thread.
+          next unless Thread.current == @thread
+
+          track_query(data[:sql].strip, data[:binds], start, finish)
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 265ef867fff165643784640d837579ce4fcc2207 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 5 Nov 2015 18:01:05 +0100
Subject: Added specs and source documentation for Sherlock

---
 lib/gitlab/sherlock/collection.rb    |  7 +++++++
 lib/gitlab/sherlock/file_sample.rb   |  4 ++++
 lib/gitlab/sherlock/line_profiler.rb | 38 +++++++++++++++++++++++++++++++++++-
 lib/gitlab/sherlock/line_sample.rb   | 16 +++++++++++++++
 lib/gitlab/sherlock/location.rb      |  4 ++++
 lib/gitlab/sherlock/middleware.rb    |  7 ++++++-
 lib/gitlab/sherlock/query.rb         | 26 ++++++++++++++++++++----
 lib/gitlab/sherlock/transaction.rb   | 29 +++++++++++++++++++++------
 8 files changed, 119 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb
index accdc6469bc..66bd6258521 100644
--- a/lib/gitlab/sherlock/collection.rb
+++ b/lib/gitlab/sherlock/collection.rb
@@ -1,5 +1,10 @@
 module Gitlab
   module Sherlock
+    # A collection of transactions recorded by Sherlock.
+    #
+    # Method calls for this class are synchronized using a mutex to allow
+    # sharing of a single Collection instance between threads (e.g. when using
+    # Puma as a webserver).
     class Collection
       include Enumerable
 
@@ -34,6 +39,8 @@ module Gitlab
         sort { |a, b| b.finished_at <=> a.finished_at }
       end
 
+      private
+
       def synchronize(&block)
         @mutex.synchronize(&block)
       end
diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb
index 7a220de9abc..8a3e1a5e5bf 100644
--- a/lib/gitlab/sherlock/file_sample.rb
+++ b/lib/gitlab/sherlock/file_sample.rb
@@ -3,6 +3,10 @@ module Gitlab
     class FileSample
       attr_reader :id, :file, :line_samples, :events, :duration
 
+      # file - The full path to the file this sample belongs to.
+      # line_samples - An array of LineSample objects.
+      # duration - The total execution time in milliseconds.
+      # events - The total amount of events.
       def initialize(file, line_samples, duration, events)
         @id = SecureRandom.uuid
         @file = file
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
index a191b1e646d..152749dcc39 100644
--- a/lib/gitlab/sherlock/line_profiler.rb
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -1,12 +1,36 @@
 module Gitlab
   module Sherlock
+    # Class for profiling code on a per line basis.
+    #
+    # The LineProfiler class can be used to profile code on per line basis
+    # without littering your code with Ruby implementation specific profiling
+    # methods.
+    #
+    # This profiler only includes samples taking longer than a given threshold
+    # and those that occur in the actual application (e.g. files from Gems are
+    # ignored).
     class LineProfiler
       # The minimum amount of time that has to be spent in a file for it to be
       # included in a list of samples.
       MINIMUM_DURATION = 10.0
 
+      # Profiles the given block.
+      #
+      # Example:
+      #
+      #     profiler = LineProfiler.new
+      #
+      #     retval, samples = profiler.profile do
+      #       "cats are amazing"
+      #     end
+      #
+      #     retval  # => "cats are amazing"
+      #     samples # => [#<Gitlab::Sherlock::FileSample ...>, ...]
+      #
+      # Returns an Array containing the block's return value and an Array of
+      # FileSample objects.
       def profile(&block)
-        if RUBY_ENGINE == 'ruby'
+        if mri?
           profile_mri(&block)
         else
           raise NotImplementedError,
@@ -14,6 +38,7 @@ module Gitlab
         end
       end
 
+      # Profiles the given block using rblineprof (MRI only).
       def profile_mri
         retval  = nil
         samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield }
@@ -24,6 +49,11 @@ module Gitlab
       end
 
       # Returns an Array of file samples based on the output of rblineprof.
+      #
+      # lineprof_stats - A Hash containing rblineprof statistics on a per file
+      #                  basis.
+      #
+      # Returns an Array of FileSample objects.
       def aggregate_rblineprof(lineprof_stats)
         samples = []
 
@@ -52,9 +82,15 @@ module Gitlab
         samples
       end
 
+      private
+
       def microsec_to_millisec(microsec)
         microsec / 1000.0
       end
+
+      def mri?
+        RUBY_ENGINE == 'ruby'
+      end
     end
   end
 end
diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb
index 38df7a88e4e..eb1948eb6d6 100644
--- a/lib/gitlab/sherlock/line_sample.rb
+++ b/lib/gitlab/sherlock/line_sample.rb
@@ -3,15 +3,31 @@ module Gitlab
     class LineSample
       attr_reader :duration, :events
 
+      # duration - The execution time in milliseconds.
+      # events - The amount of events.
       def initialize(duration, events)
         @duration = duration
         @events = events
       end
 
+      # Returns the sample duration percentage relative to the given duration.
+      #
+      # Example:
+      #
+      #     sample.duration            # => 150
+      #     sample.percentage_of(1500) # => 10.0
+      #
+      # total_duration - The total duration to compare with.
+      #
+      # Returns a float
       def percentage_of(total_duration)
         (duration.to_f / total_duration) * 100.0
       end
 
+      # Returns true if the current sample takes up the majority of the given
+      # duration.
+      #
+      # total_duration - The total duration to compare with.
       def majority_of?(total_duration)
         percentage_of(total_duration) >= 30
       end
diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb
index 8c0b77dce1a..5ac265618ad 100644
--- a/lib/gitlab/sherlock/location.rb
+++ b/lib/gitlab/sherlock/location.rb
@@ -5,15 +5,19 @@ module Gitlab
 
       SHERLOCK_DIR = File.dirname(__FILE__)
 
+      # Creates a new Location from a `Thread::Backtrace::Location`.
       def self.from_ruby_location(location)
         new(location.path, location.lineno)
       end
 
+      # path - The full path of the frame as a String.
+      # line - The line number of the frame as a Fixnum.
       def initialize(path, line)
         @path = path
         @line = line
       end
 
+      # Returns true if the current frame originated from the application.
       def application?
         @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR)
       end
diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb
index fca7be858eb..687332fc5fc 100644
--- a/lib/gitlab/sherlock/middleware.rb
+++ b/lib/gitlab/sherlock/middleware.rb
@@ -10,6 +10,7 @@ module Gitlab
         @app = app
       end
 
+      # env - A Hash containing Rack environment details.
       def call(env)
         if instrument?(env)
           call_with_instrumentation(env)
@@ -19,7 +20,7 @@ module Gitlab
       end
 
       def call_with_instrumentation(env)
-        trans = Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI'])
+        trans = transaction_from_env(env)
         retval = trans.run { @app.call(env) }
 
         Sherlock.collection.add(trans)
@@ -31,6 +32,10 @@ module Gitlab
         !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES &&
            env['REQUEST_URI'] !~ IGNORE_PATHS)
       end
+
+      def transaction_from_env(env)
+        Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI'])
+      end
     end
   end
 end
diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb
index af76e6fd2bf..4917c4ae2ac 100644
--- a/lib/gitlab/sherlock/query.rb
+++ b/lib/gitlab/sherlock/query.rb
@@ -3,6 +3,7 @@ module Gitlab
     class Query
       attr_reader :id, :query, :started_at, :finished_at, :backtrace
 
+      # SQL identifiers that should be prefixed with newlines.
       PREFIX_NEWLINE = /
         \s+(FROM
           |(LEFT|RIGHT)?INNER\s+JOIN
@@ -12,11 +13,20 @@ module Gitlab
           |GROUP\s+BY
           |ORDER\s+BY
           |LIMIT
-          |OFFSET)\s+
-      /ix
-
+          |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :<
+
+      # Creates a new Query using a String and a separate Array of bindings.
+      #
+      # query - A String containing a SQL query, optionally with numeric
+      #         placeholders (`$1`, `$2`, etc).
+      #
+      # bindings - An Array of ActiveRecord columns and their values.
+      # started_at - The start time of the query as a Time-like object.
+      # finished_at - The completion time of the query as a Time-like object.
+      #
+      # Returns a new Query object.
       def self.new_with_bindings(query, bindings, started_at, finished_at)
-        bindings.each_with_index do |(column, value), index|
+        bindings.each_with_index do |(_, value), index|
           quoted_value = ActiveRecord::Base.connection.quote(value)
 
           query = query.gsub("$#{index + 1}", quoted_value)
@@ -25,6 +35,9 @@ module Gitlab
         new(query, started_at, finished_at)
       end
 
+      # query - The SQL query as a String (without placeholders).
+      # started_at - The start time of the query as a Time-like object.
+      # finished_at - The completion time of the query as a Time-like object.
       def initialize(query, started_at, finished_at)
         @id = SecureRandom.uuid
         @query = query
@@ -39,6 +52,7 @@ module Gitlab
         end
       end
 
+      # Returns the query duration in milliseconds.
       def duration
         @duration ||= (@finished_at - @started_at) * 1000.0
       end
@@ -47,18 +61,22 @@ module Gitlab
         @id
       end
 
+      # Returns a human readable version of the query.
       def formatted_query
         @formatted_query ||= format_sql(@query)
       end
 
+      # Returns the last application frame of the backtrace.
       def last_application_frame
         @last_application_frame ||= @backtrace.find(&:application?)
       end
 
+      # Returns an Array of application frames (excluding Gems and the likes).
       def application_backtrace
         @application_backtrace ||= @backtrace.select(&:application?)
       end
 
+      # Returns the query plan as a String.
       def explain
         unless @explain
           ActiveRecord::Base.connection.transaction do
diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb
index 5cb3e86aa4e..4641f15ee33 100644
--- a/lib/gitlab/sherlock/transaction.rb
+++ b/lib/gitlab/sherlock/transaction.rb
@@ -4,11 +4,12 @@ module Gitlab
       attr_reader :id, :type, :path, :queries, :file_samples, :started_at,
         :finished_at
 
+      # type - The type of transaction (e.g. "GET", "POST", etc)
+      # path - The path of the transaction (e.g. the HTTP request path)
       def initialize(type, path)
         @id = SecureRandom.uuid
         @type = type
         @path = path
-        @duration = 0
         @queries = []
         @file_samples = []
         @started_at = nil
@@ -16,6 +17,7 @@ module Gitlab
         @thread = Thread.current
       end
 
+      # Runs the transaction and returns the block's return value.
       def run
         @started_at = Time.now
 
@@ -30,34 +32,43 @@ module Gitlab
         retval
       end
 
+      # Returns the duration in seconds.
       def duration
-        @started_at && @finished_at ? @finished_at - @started_at : 0
+        @duration ||= started_at && finished_at ? finished_at - started_at : 0
       end
 
       def to_param
         @id
       end
 
+      # Returns the queries sorted in descending order by their durations.
       def sorted_queries
         @queries.sort { |a, b| b.duration <=> a.duration }
       end
 
+      # Returns the file samples sorted in descending order by their durations.
       def sorted_file_samples
         @file_samples.sort { |a, b| b.duration <=> a.duration }
       end
 
+      # Finds a query by the given ID.
+      #
+      # id - The query ID as a String.
+      #
+      # Returns a Query object if one could be found, nil otherwise.
       def find_query(id)
         @queries.find { |query| query.id == id }
       end
 
+      # Finds a file sample by the given ID.
+      #
+      # id - The query ID as a String.
+      #
+      # Returns a FileSample object if one could be found, nil otherwise.
       def find_file_sample(id)
         @file_samples.find { |sample| sample.id == id }
       end
 
-      def track_query(query, bindings, start, finish)
-        @queries << Query.new_with_bindings(query, bindings, start, finish)
-      end
-
       def profile_lines
         retval = nil
 
@@ -70,6 +81,12 @@ module Gitlab
         retval
       end
 
+      private
+
+      def track_query(query, bindings, start, finish)
+        @queries << Query.new_with_bindings(query, bindings, start, finish)
+      end
+
       def subscribe_to_active_record
         ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
           # In case somebody uses a multi-threaded server locally (e.g. Puma) we
-- 
cgit v1.2.1


From 7b5fd8742e6112491f61f27dcca2d8e441cc33a1 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 9 Nov 2015 12:36:01 +0100
Subject: Track the amount of times views are rendered

---
 lib/gitlab/sherlock/transaction.rb | 57 ++++++++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb
index 4641f15ee33..d87a4c9bb4a 100644
--- a/lib/gitlab/sherlock/transaction.rb
+++ b/lib/gitlab/sherlock/transaction.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Sherlock
     class Transaction
       attr_reader :id, :type, :path, :queries, :file_samples, :started_at,
-        :finished_at
+        :finished_at, :view_counts
 
       # type - The type of transaction (e.g. "GET", "POST", etc)
       # path - The path of the transaction (e.g. the HTTP request path)
@@ -15,20 +15,19 @@ module Gitlab
         @started_at = nil
         @finished_at = nil
         @thread = Thread.current
+        @view_counts = Hash.new(0)
       end
 
       # Runs the transaction and returns the block's return value.
       def run
         @started_at = Time.now
 
-        subscriber = subscribe_to_active_record
-
-        retval = profile_lines { yield }
+        retval = with_subscriptions do
+          profile_lines { yield }
+        end
 
         @finished_at = Time.now
 
-        ActiveSupport::Notifications.unsubscribe(subscriber)
-
         retval
       end
 
@@ -81,21 +80,51 @@ module Gitlab
         retval
       end
 
+      def subscribe_to_active_record
+        ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
+          next unless same_thread?
+
+          track_query(data[:sql].strip, data[:binds], start, finish)
+        end
+      end
+
+      def subscribe_to_action_view
+        regex = /render_(template|partial)\.action_view/
+
+        ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data|
+          next unless same_thread?
+
+          track_view(data[:identifier])
+        end
+      end
+
       private
 
       def track_query(query, bindings, start, finish)
         @queries << Query.new_with_bindings(query, bindings, start, finish)
       end
 
-      def subscribe_to_active_record
-        ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
-          # In case somebody uses a multi-threaded server locally (e.g. Puma) we
-          # _only_ want to track queries that originate from the transaction
-          # thread.
-          next unless Thread.current == @thread
+      def track_view(path)
+        @view_counts[path] += 1
+      end
 
-          track_query(data[:sql].strip, data[:binds], start, finish)
-        end
+      def with_subscriptions
+        ar_subscriber = subscribe_to_active_record
+        av_subscriber = subscribe_to_action_view
+
+        retval = yield
+
+        ActiveSupport::Notifications.unsubscribe(ar_subscriber)
+        ActiveSupport::Notifications.unsubscribe(av_subscriber)
+
+        retval
+      end
+
+      # In case somebody uses a multi-threaded server locally (e.g. Puma) we
+      # _only_ want to track notifications that originate from the transaction
+      # thread.
+      def same_thread?
+        Thread.current == @thread
       end
     end
   end
-- 
cgit v1.2.1


From 73cf0f1647806a4ce064707c6f1f416181de48ef Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 9 Nov 2015 15:54:13 +0100
Subject: Only load rblineprof when actually needed

This ensures the application can still boot when the "development" group
is not available.
---
 lib/gitlab/sherlock.rb               | 1 -
 lib/gitlab/sherlock/line_profiler.rb | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb
index c4b35b24ceb..6360527a7aa 100644
--- a/lib/gitlab/sherlock.rb
+++ b/lib/gitlab/sherlock.rb
@@ -1,5 +1,4 @@
 require 'securerandom'
-require 'rblineprof' if RUBY_ENGINE == 'ruby'
 
 module Gitlab
   module Sherlock
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
index 152749dcc39..aa1468bff6b 100644
--- a/lib/gitlab/sherlock/line_profiler.rb
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -40,6 +40,8 @@ module Gitlab
 
       # Profiles the given block using rblineprof (MRI only).
       def profile_mri
+        require 'rblineprof'
+
         retval  = nil
         samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield }
 
-- 
cgit v1.2.1


From 1b7a2fc5363d7e2334f5c48940e8eca9d88354a6 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Tue, 10 Nov 2015 11:49:38 +0100
Subject: Improve Continuous Integration graphs page

* fix commit duration graph
* make graphs responsive
* fix wrong padding
* add a bit of explanation to colors

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/ci/charts.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index 915a4f526a6..5ff7407c6fe 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -60,7 +60,8 @@ module Ci
 
     class BuildTime < Chart
       def collect
-        commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30)
+        commits = project.commits.last(30)
+
         commits.each do |commit|
           @labels << commit.short_sha
           @build_times << (commit.duration / 60)
-- 
cgit v1.2.1


From d0e3e823a2dd56260550aec648b0cbfae64543ae Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 12 Oct 2015 23:47:32 +0200
Subject: Implement Build Artifacts

- Offloads uploading to GitLab Workhorse
- Use /authorize request for fast uploading
- Added backup recipes for artifacts
- Support download acceleration using X-Sendfile
---
 lib/api/helpers.rb                 |  44 ++++++++++++++++
 lib/backup/artifacts.rb            |  13 +++++
 lib/backup/manager.rb              |   2 +-
 lib/ci/api/api.rb                  |   1 +
 lib/ci/api/builds.rb               | 102 +++++++++++++++++++++++++++++++++++++
 lib/ci/api/entities.rb             |   7 +++
 lib/ci/api/helpers.rb              |  11 ++++
 lib/ci/gitlab_ci_yaml_processor.rb |   9 +++-
 lib/file_streamer.rb               |  16 ++++++
 lib/gitlab/current_settings.rb     |   1 +
 lib/support/nginx/gitlab           |  16 ++++++
 lib/support/nginx/gitlab-ssl       |  16 ++++++
 lib/tasks/gitlab/backup.rake       |  21 ++++++++
 lib/uploaded_file.rb               |  37 ++++++++++++++
 14 files changed, 293 insertions(+), 3 deletions(-)
 create mode 100644 lib/backup/artifacts.rb
 create mode 100644 lib/file_streamer.rb
 create mode 100644 lib/uploaded_file.rb

(limited to 'lib')

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 652bdf9b278..b980cd8391e 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -133,6 +133,12 @@ module API
       authorize! :admin_project, user_project
     end
 
+    def require_gitlab_workhorse!
+      unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present?
+        forbidden!('Request should be executed via GitLab Workhorse')
+      end
+    end
+
     def can?(object, action, subject)
       abilities.allowed?(object, action, subject)
     end
@@ -234,6 +240,10 @@ module API
       render_api_error!(message || '409 Conflict', 409)
     end
 
+    def file_to_large!
+      render_api_error!('413 Request Entity Too Large', 413)
+    end
+
     def render_validation_error!(model)
       if model.errors.any?
         render_api_error!(model.errors.messages || '400 Bad Request', 400)
@@ -282,6 +292,40 @@ module API
       end
     end
 
+    # file helpers
+
+    def uploaded_file!(uploads_path)
+      required_attributes! [:file]
+
+      # sanitize file paths
+      # this requires for all paths to exist
+      uploads_path = File.realpath(uploads_path)
+      file_path = File.realpath(params[:file])
+      bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+
+      UploadedFile.new(
+        file_path,
+        params[:filename],
+        params[:filetype] || 'application/octet-stream',
+      )
+    end
+
+    def present_file!(path, filename, content_type = 'application/octet-stream')
+      filename ||= File.basename(path)
+      header['Content-Disposition'] = "attachment; filename=#{filename}"
+      header['Content-Transfer-Encoding'] = 'binary'
+      content_type content_type
+
+      # Support download acceleration
+      case headers['X-Sendfile-Type']
+      when 'X-Sendfile'
+        header['X-Sendfile'] = path
+        body
+      else
+        file FileStreamer.new(path)
+      end
+    end
+
     private
 
     def add_pagination_headers(paginated, per_page)
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
new file mode 100644
index 00000000000..51fa3867e67
--- /dev/null
+++ b/lib/backup/artifacts.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Artifacts < Files
+    def initialize
+      super('artifacts', ArtifactUploader.artifacts_path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index f011fd03de0..9e15d5411a1 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,7 +150,7 @@ module Backup
     private
 
     def backup_contents
-      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+      folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
     end
 
     def folders_to_backup
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 0a4cbf69b63..07e68216d7f 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -27,6 +27,7 @@ module Ci
 
       helpers Helpers
       helpers ::API::Helpers
+      helpers Gitlab::CurrentSettings
 
       mount Builds
       mount Commits
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 83ca1e6481c..622849c4b11 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -47,6 +47,108 @@ module Ci
             build.drop
           end
         end
+
+        # Authorize artifacts uploading for build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        #   size (optional) - the size of uploaded file
+        # Example Request:
+        #   POST /builds/:id/artifacts/authorize
+        post ":id/artifacts/authorize" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          if params[:filesize]
+            file_size = params[:filesize].to_i
+            file_to_large! unless file_size < max_artifacts_size
+          end
+
+          status 200
+          { temp_path: ArtifactUploader.artifacts_upload_path }
+        end
+
+        # Upload artifacts to build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   Content-Type - File content type
+        #   Content-Disposition - File media type and real name
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Body:
+        #   The file content
+        #
+        # Parameters (set by GitLab Workhorse):
+        #   file - path to locally stored body (generated by Workhorse)
+        #   filename - real filename as send in Content-Disposition
+        #   filetype - real content type as send in Content-Type
+        #   filesize - real file size as send in Content-Length
+        # Example Request:
+        #   POST /builds/:id/artifacts
+        post ":id/artifacts" do
+          require_gitlab_workhorse!
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('build is not running') unless build.running?
+
+          file = uploaded_file!(ArtifactUploader.artifacts_upload_path)
+          file_to_large! unless file.size < max_artifacts_size
+
+          if build.update_attributes(artifacts_file: file)
+            present build, with: Entities::Build
+          else
+            render_validation_error!(build)
+          end
+        end
+
+        # Download the artifacts file from build - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   GET /builds/:id/artifacts
+        get ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          artifacts_file = build.artifacts_file
+
+          unless artifacts_file.file_storage?
+            return redirect_to build.artifacts_file.url
+          end
+
+          unless artifacts_file.exists?
+            not_found!
+          end
+
+          present_file!(artifacts_file.path, artifacts_file.filename)
+        end
+
+        # Remove the artifacts file from build
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        #   token (required) - The build authorization token
+        # Headers:
+        #   BUILD-TOKEN (required) - The build authorization token, the same as token
+        # Example Request:
+        #   DELETE /builds/:id/artifacts
+        delete ":id/artifacts" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          build.remove_artifacts_file!
+        end
       end
     end
   end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b80c0b8b273..750f421872d 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -11,10 +11,16 @@ module Ci
         expose :builds
       end
 
+      class ArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
       class Build < Grape::Entity
         expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
           :before_sha, :allow_git_fetch, :project_name
 
+        expose :name, :token, :stage
+
         expose :options do |model|
           model.options
         end
@@ -24,6 +30,7 @@ module Ci
         end
 
         expose :variables
+        expose :artifacts_file, using: ArtifactFile
       end
 
       class Runner < Grape::Entity
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 7e4986b6af3..02502333756 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -1,6 +1,8 @@
 module Ci
   module API
     module Helpers
+      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+      BUILD_TOKEN_PARAM = :token
       UPDATE_RUNNER_EVERY = 60
 
       def authenticate_runners!
@@ -15,6 +17,11 @@ module Ci
         forbidden! unless project.valid_token?(params[:project_token])
       end
 
+      def authenticate_build_token!(build)
+        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+        forbidden! unless token && build.valid_token?(token)
+      end
+
       def update_runner_last_contact
         # Use a random threshold to prevent beating DB updates
         contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
@@ -32,6 +39,10 @@ module Ci
         info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
         current_runner.update(info)
       end
+
+      def max_artifacts_size
+        current_application_settings.max_artifacts_size.megabytes.to_i
+      end
     end
   end
 end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 0f57a4f53ab..6f9af5388ca 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -5,7 +5,7 @@ module Ci
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
     ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
+    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
 
     attr_reader :before_script, :image, :services, :variables, :path
 
@@ -77,7 +77,8 @@ module Ci
         when: job[:when] || 'on_success',
         options: {
           image: job[:image] || @image,
-          services: job[:services] || @services
+          services: job[:services] || @services,
+          artifacts: job[:artifacts]
         }.compact
       }
     end
@@ -159,6 +160,10 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
+      if job[:artifacts] && !validate_array_of_strings(job[:artifacts])
+        raise ValidationError, "#{name}: artifacts parameter should be an array of strings"
+      end
+
       if job[:allow_failure] && !job[:allow_failure].in?([true, false])
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb
new file mode 100644
index 00000000000..4e3c6d3c773
--- /dev/null
+++ b/lib/file_streamer.rb
@@ -0,0 +1,16 @@
+class FileStreamer #:nodoc:
+  attr_reader :to_path
+
+  def initialize(path)
+    @to_path = path
+  end
+
+  # Stream the file's contents if Rack::Sendfile isn't present.
+  def each
+    File.open(to_path, 'rb') do |file|
+      while chunk = file.read(16384)
+        yield chunk
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index cd84afa31d5..2d3e32d9539 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -25,6 +25,7 @@ module Gitlab
         session_expire_delay: Settings.gitlab['session_expire_delay'],
         import_sources: Settings.gitlab['import_sources'],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+        max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
       )
     end
 
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index e767027dc29..e511d5e4b4b 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -131,6 +131,22 @@ server {
     return 418;
   }
 
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+      error_page 418 = @gitlab-git-http-server;
+      return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+      error_page 418 = @gitlab-git-http-server;
+      return 418;
+  }
+
   location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 4d31e31f8d5..47b1ec8cb0c 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -178,6 +178,22 @@ server {
     return 418;
   }
 
+  # Build artifacts should be submitted to this location
+  location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+      error_page 418 = @gitlab-git-http-server;
+      return 418;
+  }
+
+  # Build artifacts should be submitted to this location
+  location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+      client_max_body_size 0;
+      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
+      error_page 418 = @gitlab-git-http-server;
+      return 418;
+  }
+
   location @gitlab-workhorse {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index f20c7f71ba5..3c46bcea40e 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -12,6 +12,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:create"].invoke
       Rake::Task["gitlab:backup:uploads:create"].invoke
       Rake::Task["gitlab:backup:builds:create"].invoke
+      Rake::Task["gitlab:backup:artifacts:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -32,6 +33,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
       Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
+      Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -113,6 +115,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :artifacts do
+      task create: :environment do
+        $progress.puts "Dumping artifacts ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Artifacts.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring artifacts ... ".blue
+        Backup::Artifacts.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     def configure_cron_mode
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
new file mode 100644
index 00000000000..d4291f012d3
--- /dev/null
+++ b/lib/uploaded_file.rb
@@ -0,0 +1,37 @@
+require "tempfile"
+require "fileutils"
+
+# Taken from: Rack::Test::UploadedFile
+class UploadedFile
+
+  # The filename, *not* including the path, of the "uploaded" file
+  attr_reader :original_filename
+
+  # The tempfile
+  attr_reader :tempfile
+
+  # The content type of the "uploaded" file
+  attr_accessor :content_type
+
+  def initialize(path, filename, content_type = "text/plain")
+    raise "#{path} file does not exist" unless ::File.exist?(path)
+
+    @content_type = content_type
+    @original_filename = filename || ::File.basename(path)
+    @tempfile = File.new(path, 'rb')
+  end
+
+  def path
+    @tempfile.path
+  end
+
+  alias_method :local_path, :path
+
+  def method_missing(method_name, *args, &block) #:nodoc:
+    @tempfile.__send__(method_name, *args, &block)
+  end
+
+  def respond_to?(method_name, include_private = false) #:nodoc:
+    @tempfile.respond_to?(method_name, include_private) || super
+  end
+end
-- 
cgit v1.2.1


From 97f58bae87dfcfb36d5a7a490b1c0983435a19f4 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 12:18:00 +0100
Subject: Change artifacts syntax to allow uploading untracked files

---
 lib/ci/gitlab_ci_yaml_processor.rb | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 6f9af5388ca..2e2209031ee 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -160,11 +160,17 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
-      if job[:artifacts] && !validate_array_of_strings(job[:artifacts])
-        raise ValidationError, "#{name}: artifacts parameter should be an array of strings"
+      if job[:artifacts]
+        if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
+          raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
+        end
+
+        if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
+          raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
+        end
       end
 
-      if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+      if job[:allow_failure] && !validate_boolean(job[:allow_failure])
         raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
       end
 
@@ -187,6 +193,10 @@ module Ci
       value.is_a?(String) || value.is_a?(Symbol)
     end
 
+    def validate_boolean(value)
+      value.in?([true, false])
+    end
+
     def process?(only_params, except_params, ref, tag)
       if only_params.present?
         return false unless matching?(only_params, ref, tag)
-- 
cgit v1.2.1


From 445cdb7579792d0d76c2562b971583bd7d05429b Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 12:49:47 +0100
Subject: Move tmp artifacts to shared/artifacts/tmp/. Check for
 GitLab-Workhorse now

---
 lib/api/helpers.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index b980cd8391e..077537959d7 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -134,7 +134,7 @@ module API
     end
 
     def require_gitlab_workhorse!
-      unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present?
+      unless env['HTTP_GITLAB_WORKHORSE'].present?
         forbidden!('Request should be executed via GitLab Workhorse')
       end
     end
-- 
cgit v1.2.1


From 7e4e3fb3b6177d2863580786e9ef6ee3a4ccf037 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 12:55:32 +0100
Subject: Fix nginx config to use @gitlab-workhorse

---
 lib/support/nginx/gitlab     | 8 ++++----
 lib/support/nginx/gitlab-ssl | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index e511d5e4b4b..0a7a4118077 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -134,16 +134,16 @@ server {
   # Build artifacts should be submitted to this location
   location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
       client_max_body_size 0;
-      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-      error_page 418 = @gitlab-git-http-server;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
       return 418;
   }
 
   # Build artifacts should be submitted to this location
   location ~ /ci/api/v1/builds/[0-9]+/artifacts {
       client_max_body_size 0;
-      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-      error_page 418 = @gitlab-git-http-server;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
       return 418;
   }
 
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 47b1ec8cb0c..b463d5b6aa9 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -181,16 +181,16 @@ server {
   # Build artifacts should be submitted to this location
   location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
       client_max_body_size 0;
-      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-      error_page 418 = @gitlab-git-http-server;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
       return 418;
   }
 
   # Build artifacts should be submitted to this location
   location ~ /ci/api/v1/builds/[0-9]+/artifacts {
       client_max_body_size 0;
-      # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
-      error_page 418 = @gitlab-git-http-server;
+      # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+      error_page 418 = @gitlab-workhorse;
       return 418;
   }
 
-- 
cgit v1.2.1


From db3213fc1c653b20783f9a41074eaf17132010de Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 22:01:26 +0100
Subject: Use normal file upload mechanism to upload artifacts

---
 lib/api/helpers.rb   | 16 ++++++++++------
 lib/ci/api/builds.rb | 16 +++++++---------
 2 files changed, 17 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 077537959d7..92540ccf2b1 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -294,19 +294,23 @@ module API
 
     # file helpers
 
-    def uploaded_file!(uploads_path)
-      required_attributes! [:file]
+    def uploaded_file!(field, uploads_path)
+      if params[field]
+        bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
+        return params[field]
+      end
 
       # sanitize file paths
-      # this requires for all paths to exist
+      # this requires all paths to exist
+      required_attributes! %W(#{field}.path)
       uploads_path = File.realpath(uploads_path)
-      file_path = File.realpath(params[:file])
+      file_path = File.realpath(params["#{field}.path"])
       bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
 
       UploadedFile.new(
         file_path,
-        params[:filename],
-        params[:filetype] || 'application/octet-stream',
+        params["#{field}.name"],
+        params["#{field}.type"] || 'application/octet-stream',
       )
     end
 
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 622849c4b11..dab0df12635 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -53,7 +53,7 @@ module Ci
         # Parameters:
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
-        #   size (optional) - the size of uploaded file
+        #   filesize (optional) - the size of uploaded file
         # Example Request:
         #   POST /builds/:id/artifacts/authorize
         post ":id/artifacts/authorize" do
@@ -77,18 +77,16 @@ module Ci
         # Parameters:
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
+        #   file (required) - The uploaded file
+        # Parameters (accelerated by GitLab Workhorse):
+        #   file.path - path to locally stored body (generated by Workhorse)
+        #   file.name - real filename as send in Content-Disposition
+        #   file.type - real content type as send in Content-Type
         # Headers:
-        #   Content-Type - File content type
-        #   Content-Disposition - File media type and real name
         #   BUILD-TOKEN (required) - The build authorization token, the same as token
         # Body:
         #   The file content
         #
-        # Parameters (set by GitLab Workhorse):
-        #   file - path to locally stored body (generated by Workhorse)
-        #   filename - real filename as send in Content-Disposition
-        #   filetype - real content type as send in Content-Type
-        #   filesize - real file size as send in Content-Length
         # Example Request:
         #   POST /builds/:id/artifacts
         post ":id/artifacts" do
@@ -98,7 +96,7 @@ module Ci
           authenticate_build_token!(build)
           forbidden!('build is not running') unless build.running?
 
-          file = uploaded_file!(ArtifactUploader.artifacts_upload_path)
+          file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
           file_to_large! unless file.size < max_artifacts_size
 
           if build.update_attributes(artifacts_file: file)
-- 
cgit v1.2.1


From 58429d9b26b2f2b62fecab012fce0ebe36571129 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Tue, 10 Nov 2015 18:34:05 +0100
Subject: Add method complexity check to CI

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/flog.rake | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 lib/tasks/flog.rake

(limited to 'lib')

diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake
new file mode 100644
index 00000000000..4cb19c0c937
--- /dev/null
+++ b/lib/tasks/flog.rake
@@ -0,0 +1,24 @@
+desc 'Code complexity analyze via flog'
+task :flog do
+  output = %x(bundle exec flog -m app/ lib/gitlab)
+  exit_code = 0
+  output = output.lines
+
+  # Skip total complexity score
+  output.shift
+
+  # Skip some trash info
+  output.shift
+
+  output.each do |line|
+    score, method = line.split(" ")
+    score = score.to_i
+
+    if score > 40
+      exit_code = 1
+      puts "High complexity in #{method}. Score: #{score}"
+    end
+  end
+
+  exit exit_code
+end
-- 
cgit v1.2.1


From eef129bd4ad41665cf5c1f1631e80b2f1bb3e4af Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 10 Nov 2015 22:21:15 +0100
Subject: Final fixes

---
 lib/ci/api/builds.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index dab0df12635..0a586672807 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -69,7 +69,7 @@ module Ci
           end
 
           status 200
-          { temp_path: ArtifactUploader.artifacts_upload_path }
+          { TempPath: ArtifactUploader.artifacts_upload_path }
         end
 
         # Upload artifacts to build - Runners only
-- 
cgit v1.2.1


From 58074ab7da9c24c030054e6303f9020f1d1f6f83 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Tue, 10 Nov 2015 22:48:13 +0100
Subject: Allow to define cache in `.gitlab-ci.yml`

---
 lib/ci/gitlab_ci_yaml_processor.rb | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 2e2209031ee..3beafcad117 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -4,10 +4,10 @@ module Ci
 
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
-    ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
+    ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache]
+    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache]
 
-    attr_reader :before_script, :image, :services, :variables, :path
+    attr_reader :before_script, :image, :services, :variables, :path, :cache
 
     def initialize(config, path = nil)
       @config = YAML.load(config)
@@ -46,6 +46,7 @@ module Ci
       @services = @config[:services]
       @stages = @config[:stages] || @config[:types]
       @variables = @config[:variables] || {}
+      @cache = @config[:cache]
       @config.except!(*ALLOWED_YAML_KEYS)
 
       # anything that doesn't have script is considered as unknown
@@ -78,7 +79,8 @@ module Ci
         options: {
           image: job[:image] || @image,
           services: job[:services] || @services,
-          artifacts: job[:artifacts]
+          artifacts: job[:artifacts],
+          cache: job[:cache] || @cache,
         }.compact
       }
     end
@@ -112,6 +114,16 @@ module Ci
         raise ValidationError, "variables should be a map of key-valued strings"
       end
 
+      if @cache
+        if @cache[:untracked] && !validate_boolean(@cache[:untracked])
+          raise ValidationError, "cache:untracked parameter should be an boolean"
+        end
+
+        if @cache[:paths] && !validate_array_of_strings(@cache[:paths])
+          raise ValidationError, "cache:paths parameter should be an array of strings"
+        end
+      end
+
       @jobs.each do |name, job|
         validate_job!(name, job)
       end
@@ -160,6 +172,16 @@ module Ci
         raise ValidationError, "#{name} job: except parameter should be an array of strings"
       end
 
+      if job[:cache]
+        if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
+          raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
+        end
+
+        if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths])
+          raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings"
+        end
+      end
+
       if job[:artifacts]
         if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
           raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
-- 
cgit v1.2.1


From 6748dd2fd0b2cf694661e910c6c1bb6ac2be69a1 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Wed, 11 Nov 2015 15:30:23 +0100
Subject: Allow flog failure for now

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/flog.rake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake
index 4cb19c0c937..3bfe999ae74 100644
--- a/lib/tasks/flog.rake
+++ b/lib/tasks/flog.rake
@@ -2,6 +2,7 @@ desc 'Code complexity analyze via flog'
 task :flog do
   output = %x(bundle exec flog -m app/ lib/gitlab)
   exit_code = 0
+  minimum_score = 70
   output = output.lines
 
   # Skip total complexity score
@@ -14,7 +15,7 @@ task :flog do
     score, method = line.split(" ")
     score = score.to_i
 
-    if score > 40
+    if score > minimum_score
       exit_code = 1
       puts "High complexity in #{method}. Score: #{score}"
     end
-- 
cgit v1.2.1


From 98c954ee815823f27cf021b34684c2c3d5bb0917 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Wed, 11 Nov 2015 16:29:00 +0100
Subject: Add flay: tool to find duplicate code

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/flay.rake | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 lib/tasks/flay.rake

(limited to 'lib')

diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
new file mode 100644
index 00000000000..5efffc2cdac
--- /dev/null
+++ b/lib/tasks/flay.rake
@@ -0,0 +1,9 @@
+desc 'Code duplication analyze via flay'
+task :flay do
+  output = %x(bundle exec flay app/ lib/gitlab/)
+
+  if output.include? "Similar code found"
+    puts output
+    exit 1
+  end
+end
-- 
cgit v1.2.1


From a5ab56fd9191e23dfa60707fa42802342c1563f8 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Thu, 12 Nov 2015 15:41:13 +0100
Subject: Move git tags API to separate file

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/api/api.rb          |  1 +
 lib/api/repositories.rb | 35 -----------------------------------
 lib/api/tags.rb         | 44 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+), 35 deletions(-)
 create mode 100644 lib/api/tags.rb

(limited to 'lib')

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 40671e2517c..fe1bf8a4816 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -52,5 +52,6 @@ module API
     mount Labels
     mount Settings
     mount Keys
+    mount Tags
   end
 end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 20d568cf462..d7c48639eba 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -16,41 +16,6 @@ module API
         end
       end
 
-      # Get a project repository tags
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id/repository/tags
-      get ":id/repository/tags" do
-        present user_project.repo.tags.sort_by(&:name).reverse,
-                with: Entities::RepoTag, project: user_project
-      end
-
-      # Create tag
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   tag_name (required) - The name of the tag
-      #   ref (required) - Create tag from commit sha or branch
-      #   message (optional) - Specifying a message creates an annotated tag.
-      # Example Request:
-      #   POST /projects/:id/repository/tags
-      post ':id/repository/tags' do
-        authorize_push_project
-        message = params[:message] || nil
-        result = CreateTagService.new(user_project, current_user).
-          execute(params[:tag_name], params[:ref], message)
-
-        if result[:status] == :success
-          present result[:tag],
-                  with: Entities::RepoTag,
-                  project: user_project
-        else
-          render_api_error!(result[:message], 400)
-        end
-      end
-
       # Get a project repository tree
       #
       # Parameters:
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
new file mode 100644
index 00000000000..da962bd402a
--- /dev/null
+++ b/lib/api/tags.rb
@@ -0,0 +1,44 @@
+module API
+  # Releases API
+  class Tags < Grape::API
+    before { authenticate! }
+    before { authorize! :download_code, user_project }
+
+    resource :projects do
+      # Get a project repository tags
+      #
+      # Parameters:
+      #   id (required) - The ID of a project
+      # Example Request:
+      #   GET /projects/:id/repository/tags
+      get ":id/repository/tags" do
+        present user_project.repo.tags.sort_by(&:name).reverse,
+                with: Entities::RepoTag, project: user_project
+      end
+
+      # Create tag
+      #
+      # Parameters:
+      #   id (required) - The ID of a project
+      #   tag_name (required) - The name of the tag
+      #   ref (required) - Create tag from commit sha or branch
+      #   message (optional) - Specifying a message creates an annotated tag.
+      # Example Request:
+      #   POST /projects/:id/repository/tags
+      post ':id/repository/tags' do
+        authorize_push_project
+        message = params[:message] || nil
+        result = CreateTagService.new(user_project, current_user).
+          execute(params[:tag_name], params[:ref], message)
+
+        if result[:status] == :success
+          present result[:tag],
+                  with: Entities::RepoTag,
+                  project: user_project
+        else
+          render_api_error!(result[:message], 400)
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From d343d9d8c2f3db0ca5f6fab8ad19e0f79f66e138 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Thu, 12 Nov 2015 16:26:23 +0100
Subject: Add grape routing print

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/grape.rake | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 lib/tasks/grape.rake

(limited to 'lib')

diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake
new file mode 100644
index 00000000000..9980e0b7984
--- /dev/null
+++ b/lib/tasks/grape.rake
@@ -0,0 +1,8 @@
+namespace :grape do
+  desc 'Print compiled grape routes'
+  task routes: :environment do
+    API::API.routes.each do |route|
+      puts route
+    end
+  end
+end
-- 
cgit v1.2.1


From c119a737935f1e1e8ae7dec814be8accb8980e1d Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Thu, 12 Nov 2015 16:26:39 +0100
Subject: Add releases api

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/api/entities.rb |  4 ++++
 lib/api/tags.rb     | 19 ++++++++++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 20cadae2291..400900bc407 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -341,5 +341,9 @@ module API
       expose :user_oauth_applications
       expose :after_sign_out_path
     end
+
+    class Release < Grape::Entity
+      expose :tag, :description
+    end
   end
 end
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index da962bd402a..06fa1d23fd6 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -1,5 +1,5 @@
 module API
-  # Releases API
+  # Git Tags API
   class Tags < Grape::API
     before { authenticate! }
     before { authorize! :download_code, user_project }
@@ -39,6 +39,23 @@ module API
           render_api_error!(result[:message], 400)
         end
       end
+
+      # Add release notes to tag
+      #
+      # Parameters:
+      #   id (required) - The ID of a project
+      #   tag (required) - The name of the tag
+      #   description (required) - Release notes with markdown support
+      # Example Request:
+      #   PUT /projects/:id/repository/tags
+      put ':id/repository/:tag/release', requirements: { tag: /.*/ } do
+        authorize_push_project
+        required_attributes! [:description]
+        release = user_project.releases.find_or_initialize_by(tag: params[:tag])
+        release.update_attributes(description: params[:description])
+
+        present release, with: Entities::Release
+      end
     end
   end
 end
-- 
cgit v1.2.1


From ba34045f3120fc52ce714d49778a24c647248500 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Thu, 12 Nov 2015 17:04:18 +0100
Subject: Expose release notes to tags api

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/api/entities.rb | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'lib')

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 400900bc407..7528e718c6f 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -112,6 +112,12 @@ module API
           options[:project].repository.commit(repo_obj.target)
         end
       end
+
+      expose :release do |repo_obj, options|
+        if options[:project]
+          options[:project].releases.find_by(tag: repo_obj.name)
+        end
+      end
     end
 
     class RepoObject < Grape::Entity
-- 
cgit v1.2.1


From 8f53094f0f22fe208f6f6e0d39037188bf14a937 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Thu, 12 Nov 2015 23:52:02 +0100
Subject: Add API docs and correctly expose release api

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/api/entities.rb | 50 +++++++++++++++++++++++++-------------------------
 lib/api/tags.rb     |  2 +-
 2 files changed, 26 insertions(+), 26 deletions(-)

(limited to 'lib')

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 7528e718c6f..3ca632dfae9 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -95,31 +95,6 @@ module API
       end
     end
 
-    class RepoTag < Grape::Entity
-      expose :name
-      expose :message do |repo_obj, _options|
-        if repo_obj.respond_to?(:message)
-          repo_obj.message
-        else
-          nil
-        end
-      end
-
-      expose :commit do |repo_obj, options|
-        if repo_obj.respond_to?(:commit)
-          repo_obj.commit
-        elsif options[:project]
-          options[:project].repository.commit(repo_obj.target)
-        end
-      end
-
-      expose :release do |repo_obj, options|
-        if options[:project]
-          options[:project].releases.find_by(tag: repo_obj.name)
-        end
-      end
-    end
-
     class RepoObject < Grape::Entity
       expose :name
 
@@ -351,5 +326,30 @@ module API
     class Release < Grape::Entity
       expose :tag, :description
     end
+
+    class RepoTag < Grape::Entity
+      expose :name
+      expose :message do |repo_obj, _options|
+        if repo_obj.respond_to?(:message)
+          repo_obj.message
+        else
+          nil
+        end
+      end
+
+      expose :commit do |repo_obj, options|
+        if repo_obj.respond_to?(:commit)
+          repo_obj.commit
+        elsif options[:project]
+          options[:project].repository.commit(repo_obj.target)
+        end
+      end
+
+      expose :release, using: Entities::Release do |repo_obj, options|
+        if options[:project]
+          options[:project].releases.find_by(tag: repo_obj.name)
+        end
+      end
+    end
   end
 end
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 06fa1d23fd6..673342dd447 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -29,7 +29,7 @@ module API
         authorize_push_project
         message = params[:message] || nil
         result = CreateTagService.new(user_project, current_user).
-          execute(params[:tag_name], params[:ref], message)
+          execute(params[:tag_name], params[:ref], message, params[:release_description])
 
         if result[:status] == :success
           present result[:tag],
-- 
cgit v1.2.1


From ea6a78ef5cb665d6f3b06d71fa693ffa895b4b1a Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Fri, 13 Nov 2015 00:44:16 +0100
Subject: Split complex Gitlab::InlineDiff::processing method

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/inline_diff.rb | 55 +++++++++++++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 19 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb
index 99e7b529ba9..420c3099ce7 100644
--- a/lib/gitlab/inline_diff.rb
+++ b/lib/gitlab/inline_diff.rb
@@ -16,15 +16,7 @@ module Gitlab
           # Skip inline diff if empty line was replaced with content
           next if first_line == "-\n"
 
-          first_the_same_symbols = 0
-          (0..max_length + 1).each do |i|
-            first_the_same_symbols = i - 1
-            if first_line[i] != second_line[i] && i > 0
-              break
-            end
-          end
-
-          first_token = first_line[0..first_the_same_symbols][1..-1]
+          first_token = find_first_token(first_line, second_line)
           start = first_token + START
 
           if first_token.empty?
@@ -36,25 +28,50 @@ module Gitlab
             diff_arr[index+2].sub!(first_token, first_token => start)
           end
 
-          last_the_same_symbols = 0
-          (1..max_length + 1).each do |i|
-            last_the_same_symbols = -i
-            shortest_line = second_line.size > first_line.size ? first_line : second_line
-            if ( first_line[-i] != second_line[-i] ) || "#{first_token}#{START}".size == shortest_line[1..-i].size
-              break
-            end
-          end
-          last_the_same_symbols += 1
-          last_token = first_line[last_the_same_symbols..-1]
+          last_token = find_last_token(first_line, second_line, first_token)
+
           # This is tricky: escape backslashes so that `sub` doesn't interpret them
           # as backreferences. Regexp.escape does NOT do the right thing.
           replace_token = FINISH + last_token.gsub(/\\/, '\&\&')
           diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
           diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
         end
+
         diff_arr
       end
 
+      def find_first_token(first_line, second_line)
+        max_length = [first_line.size, second_line.size].max
+        first_the_same_symbols = 0
+
+        (0..max_length + 1).each do |i|
+          first_the_same_symbols = i - 1
+
+          if first_line[i] != second_line[i] && i > 0
+            break
+          end
+        end
+
+        first_line[0..first_the_same_symbols][1..-1]
+      end
+
+      def find_last_token(first_line, second_line, first_token)
+        max_length = [first_line.size, second_line.size].max
+        last_the_same_symbols = 0
+
+        (1..max_length + 1).each do |i|
+          last_the_same_symbols = -i
+          shortest_line = second_line.size > first_line.size ? first_line : second_line
+
+          if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size
+            break
+          end
+        end
+
+        last_the_same_symbols += 1
+        first_line[last_the_same_symbols..-1]
+      end
+
       def _indexes_of_changed_lines(diff_arr)
         chain_of_first_symbols = ""
         diff_arr.each_with_index do |line, i|
-- 
cgit v1.2.1


From c903f23f3c0b5654c2883cdf63593e53a8898a79 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Fri, 13 Nov 2015 00:55:20 +0100
Subject: Split complex methods in GoogleCodeImport::Importer

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/google_code_import/importer.rb | 93 ++++++++++++++++---------------
 1 file changed, 47 insertions(+), 46 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb
index 03c410726a5..87fee28dc01 100644
--- a/lib/gitlab/google_code_import/importer.rb
+++ b/lib/gitlab/google_code_import/importer.rb
@@ -30,7 +30,7 @@ module Gitlab
 
       def user_map
         @user_map ||= begin
-          user_map = Hash.new do |hash, user| 
+          user_map = Hash.new do |hash, user|
             # Replace ... by \.\.\., so `johnsm...@gmail.com` isn't autolinked.
             Client.mask_email(user).sub("...", "\\.\\.\\.")
           end
@@ -76,18 +76,7 @@ module Gitlab
           attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"])
 
           body = format_issue_body(author, date, content, attachments)
-
-          labels = []
-          raw_issue["labels"].each do |label|
-            name = nice_label_name(label)
-            labels << name
-
-            unless @known_labels.include?(name)
-              create_label(name)
-              @known_labels << name
-            end
-          end
-          labels << nice_status_name(raw_issue["status"])
+          labels = import_issue_labels(raw_issue)
 
           assignee_id = nil
           if raw_issue.has_key?("owner")
@@ -110,6 +99,7 @@ module Gitlab
             assignee_id:  assignee_id,
             state:        raw_issue["state"] == "closed" ? "closed" : "opened"
           )
+
           issue.add_labels_by_names(labels)
 
           if issue.iid != raw_issue["id"]
@@ -120,6 +110,23 @@ module Gitlab
         end
       end
 
+      def import_issue_labels(raw_issue)
+        labels = []
+
+        raw_issue["labels"].each do |label|
+          name = nice_label_name(label)
+          labels << name
+
+          unless @known_labels.include?(name)
+            create_label(name)
+            @known_labels << name
+          end
+        end
+
+        labels << nice_status_name(raw_issue["status"])
+        labels
+      end
+
       def import_issue_comments(issue, comments)
         Note.transaction do
           while raw_comment = comments.shift
@@ -172,7 +179,7 @@ module Gitlab
           "#5cb85c"
         when "Status: Started"
           "#8e44ad"
-        
+
         when "Priority: Critical"
           "#ffcfcf"
         when "Priority: High"
@@ -181,7 +188,7 @@ module Gitlab
           "#fff5cc"
         when "Priority: Low"
           "#cfe9ff"
-        
+
         when "Type: Defect"
           "#d9534f"
         when "Type: Enhancement"
@@ -249,8 +256,8 @@ module Gitlab
         end
 
         if raw_updates.has_key?("cc")
-          cc = raw_updates["cc"].map do |l| 
-            deleted = l.start_with?("-") 
+          cc = raw_updates["cc"].map do |l|
+            deleted = l.start_with?("-")
             l = l[1..-1] if deleted
             l = user_map[l]
             l = "~~#{l}~~" if deleted
@@ -261,8 +268,8 @@ module Gitlab
         end
 
         if raw_updates.has_key?("labels")
-          labels = raw_updates["labels"].map do |l| 
-            deleted = l.start_with?("-") 
+          labels = raw_updates["labels"].map do |l|
+            deleted = l.start_with?("-")
             l = l[1..-1] if deleted
             l = nice_label_name(l)
             l = "~~#{l}~~" if deleted
@@ -278,45 +285,39 @@ module Gitlab
 
         if raw_updates.has_key?("blockedOn")
           blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on|
-            name, id = raw_blocked_on.split(":", 2)
-
-            deleted = name.start_with?("-") 
-            name = name[1..-1] if deleted
-
-            text =
-              if name == project.import_source
-                "##{id}"
-              else
-                "#{project.namespace.path}/#{name}##{id}"
-              end
-            text = "~~#{text}~~" if deleted
-            text
+            format_blocking_updates(raw_blocked_on)
           end
+
           updates << "*Blocked on: #{blocked_ons.join(", ")}*"
         end
 
         if raw_updates.has_key?("blocking")
           blockings = raw_updates["blocking"].map do |raw_blocked_on|
-            name, id = raw_blocked_on.split(":", 2)
-            
-            deleted = name.start_with?("-") 
-            name = name[1..-1] if deleted
-
-            text =
-              if name == project.import_source
-                "##{id}"
-              else
-                "#{project.namespace.path}/#{name}##{id}"
-              end
-            text = "~~#{text}~~" if deleted
-            text
+            format_blocking_updates(raw_blocked_on)
           end
+
           updates << "*Blocking: #{blockings.join(", ")}*"
         end
 
         updates
       end
 
+      def format_blocking_updates(raw_blocked_on)
+        name, id = raw_blocked_on.split(":", 2)
+
+        deleted = name.start_with?("-")
+        name = name[1..-1] if deleted
+
+        text =
+          if name == project.import_source
+            "##{id}"
+          else
+            "#{project.namespace.path}/#{name}##{id}"
+          end
+        text = "~~#{text}~~" if deleted
+        text
+      end
+
       def format_attachments(issue_id, comment_id, raw_attachments)
         return [] unless raw_attachments
 
@@ -325,7 +326,7 @@ module Gitlab
 
           filename = attachment["fileName"]
           link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}"
-          
+
           text = "[#{filename}](#{link})"
           text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/i
           text
-- 
cgit v1.2.1


From c71892e1a87ece19edf17f1d61c71d4b99953bc7 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Fri, 13 Nov 2015 01:07:33 +0100
Subject: Even more refactoring to inline_diff.rb

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/inline_diff.rb | 39 +++++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 16 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb
index 420c3099ce7..72d0b493be0 100644
--- a/lib/gitlab/inline_diff.rb
+++ b/lib/gitlab/inline_diff.rb
@@ -17,29 +17,36 @@ module Gitlab
           next if first_line == "-\n"
 
           first_token = find_first_token(first_line, second_line)
-          start = first_token + START
-
-          if first_token.empty?
-            # In case if we remove string of spaces in commit
-            diff_arr[index+1].sub!("-", "-" => "-#{START}")
-            diff_arr[index+2].sub!("+", "+" => "+#{START}")
-          else
-            diff_arr[index+1].sub!(first_token, first_token => start)
-            diff_arr[index+2].sub!(first_token, first_token => start)
-          end
+          apply_first_token(diff_arr, index, first_token)
 
           last_token = find_last_token(first_line, second_line, first_token)
-
-          # This is tricky: escape backslashes so that `sub` doesn't interpret them
-          # as backreferences. Regexp.escape does NOT do the right thing.
-          replace_token = FINISH + last_token.gsub(/\\/, '\&\&')
-          diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
-          diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
+          apply_last_token(diff_arr, index, last_token)
         end
 
         diff_arr
       end
 
+      def apply_first_token(diff_arr, index, first_token)
+        start = first_token + START
+
+        if first_token.empty?
+          # In case if we remove string of spaces in commit
+          diff_arr[index+1].sub!("-", "-" => "-#{START}")
+          diff_arr[index+2].sub!("+", "+" => "+#{START}")
+        else
+          diff_arr[index+1].sub!(first_token, first_token => start)
+          diff_arr[index+2].sub!(first_token, first_token => start)
+        end
+      end
+
+      def apply_last_token(diff_arr, index, last_token)
+        # This is tricky: escape backslashes so that `sub` doesn't interpret them
+        # as backreferences. Regexp.escape does NOT do the right thing.
+        replace_token = FINISH + last_token.gsub(/\\/, '\&\&')
+        diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
+        diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
+      end
+
       def find_first_token(first_line, second_line)
         max_length = [first_line.size, second_line.size].max
         first_the_same_symbols = 0
-- 
cgit v1.2.1


From 3d0efa8e0a359c84485a0fd7a3317745bf5648b8 Mon Sep 17 00:00:00 2001
From: Minsik Yoon <yms9654@gmail.com>
Date: Thu, 22 Oct 2015 09:55:35 +0900
Subject: Add ignore white space option in merge request diff

fix this issue(https://gitlab.com/gitlab-org/gitlab-ce/issues/1393).

Add ignore whitespace optoin to Commits Compare view
---
 lib/gitlab/compare_result.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb
index d72391dade5..0d696a1ee28 100644
--- a/lib/gitlab/compare_result.rb
+++ b/lib/gitlab/compare_result.rb
@@ -2,8 +2,8 @@ module Gitlab
   class CompareResult
     attr_reader :commits, :diffs
 
-    def initialize(compare)
-      @commits, @diffs = compare.commits, compare.diffs
+    def initialize(compare, diff_options = {})
+      @commits, @diffs = compare.commits, compare.diffs(nil, diff_options)
     end
   end
 end
-- 
cgit v1.2.1


From 3c16fb93fcfb7dc50a46a39cbdd545ddfdab0bc1 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Fri, 13 Nov 2015 09:45:21 +0100
Subject: Move spec to proper place and fix unused variable

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/inline_diff.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb
index 72d0b493be0..44507bde25d 100644
--- a/lib/gitlab/inline_diff.rb
+++ b/lib/gitlab/inline_diff.rb
@@ -11,7 +11,6 @@ module Gitlab
         indexes.each do |index|
           first_line = diff_arr[index+1]
           second_line = diff_arr[index+2]
-          max_length = [first_line.size, second_line.size].max
 
           # Skip inline diff if empty line was replaced with content
           next if first_line == "-\n"
-- 
cgit v1.2.1


From 6384c757b7ce6d1c0c3e2a3828b0cfac26dfb7f9 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Mon, 9 Nov 2015 16:48:03 +0100
Subject: Expose CI enable option in project features

- Enable CI by default for all new projects
---
 lib/api/entities.rb | 2 +-
 lib/api/projects.rb | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 20cadae2291..73acf66935a 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -62,7 +62,7 @@ module API
       expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group }
       expose :name, :name_with_namespace
       expose :path, :path_with_namespace
-      expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :snippets_enabled, :created_at, :last_activity_at
+      expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at
       expose :creator_id
       expose :namespace
       expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ | project, options | project.forked? }
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 67ee66a2058..2b4ada6e2eb 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -75,6 +75,7 @@ module API
       #   description (optional) - short project description
       #   issues_enabled (optional)
       #   merge_requests_enabled (optional)
+      #   builds_enabled (optional)
       #   wiki_enabled (optional)
       #   snippets_enabled (optional)
       #   namespace_id (optional) - defaults to user namespace
@@ -90,6 +91,7 @@ module API
                                      :description,
                                      :issues_enabled,
                                      :merge_requests_enabled,
+                                     :builds_enabled,
                                      :wiki_enabled,
                                      :snippets_enabled,
                                      :namespace_id,
@@ -117,6 +119,7 @@ module API
       #   default_branch (optional) - 'master' by default
       #   issues_enabled (optional)
       #   merge_requests_enabled (optional)
+      #   builds_enabled (optional)
       #   wiki_enabled (optional)
       #   snippets_enabled (optional)
       #   public (optional) - if true same as setting visibility_level = 20
@@ -132,6 +135,7 @@ module API
                                      :default_branch,
                                      :issues_enabled,
                                      :merge_requests_enabled,
+                                     :builds_enabled,
                                      :wiki_enabled,
                                      :snippets_enabled,
                                      :public,
@@ -172,6 +176,7 @@ module API
       #   description (optional) - short project description
       #   issues_enabled (optional)
       #   merge_requests_enabled (optional)
+      #   builds_enabled (optional)
       #   wiki_enabled (optional)
       #   snippets_enabled (optional)
       #   public (optional) - if true same as setting visibility_level = 20
@@ -185,6 +190,7 @@ module API
                                      :default_branch,
                                      :issues_enabled,
                                      :merge_requests_enabled,
+                                     :builds_enabled,
                                      :wiki_enabled,
                                      :snippets_enabled,
                                      :public,
-- 
cgit v1.2.1


From 4bb99677b1ddee84ab0433efe081de8025e5aa53 Mon Sep 17 00:00:00 2001
From: Alec Cooper <ahnolds@gmail.com>
Date: Thu, 12 Nov 2015 21:38:26 -0500
Subject: Relative links in the README file shown on the repository homepage
 should point to the default branch, not to master

---
 lib/gitlab/markdown/relative_link_filter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb
index 6ee3d1ce039..632be4d7542 100644
--- a/lib/gitlab/markdown/relative_link_filter.rb
+++ b/lib/gitlab/markdown/relative_link_filter.rb
@@ -51,7 +51,7 @@ module Gitlab
           relative_url_root,
           context[:project].path_with_namespace,
           path_type(file_path),
-          ref || 'master',  # assume that if no ref exists we can point to master
+          ref || context[:project].default_branch,  # if no ref exists, point to the default branch
           file_path
         ].compact.join('/').squeeze('/').chomp('/')
 
-- 
cgit v1.2.1


From 14032d8eb1a60ae5920286249c1044be2fa27278 Mon Sep 17 00:00:00 2001
From: Marin Jankovski <maxlazio@gmail.com>
Date: Mon, 12 Oct 2015 16:42:14 +0200
Subject: Add support for git lfs.

---
 lib/gitlab/backend/grack_auth.rb |   5 +-
 lib/gitlab/git_access.rb         |   6 +-
 lib/gitlab/lfs/response.rb       | 308 +++++++++++++++++++++++++++++++++++++++
 lib/gitlab/lfs/router.rb         |  95 ++++++++++++
 lib/support/nginx/gitlab         |   9 +-
 lib/support/nginx/gitlab-ssl     |   9 +-
 6 files changed, 426 insertions(+), 6 deletions(-)
 create mode 100644 lib/gitlab/lfs/response.rb
 create mode 100644 lib/gitlab/lfs/router.rb

(limited to 'lib')

diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 440ef5a3cb3..0d156047ff0 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -33,6 +33,9 @@ module Grack
 
       auth!
 
+      lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call
+      return lfs_response unless lfs_response.nil?
+
       if project && authorized_request?
         # Tell gitlab-workhorse the request is OK, and what the GL_ID is
         render_grack_auth_ok
@@ -72,7 +75,7 @@ module Grack
       matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login)
 
       if project && matched_login.present? && git_cmd == 'git-upload-pack'
-        underscored_service = matched_login['s'].underscore 
+        underscored_service = matched_login['s'].underscore
 
         if Service.available_services_names.include?(underscored_service)
           service_method = "#{underscored_service}_service"
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index c90184d31cf..3ed1eec517c 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -13,7 +13,7 @@ module Gitlab
     def user
       return @user if defined?(@user)
 
-      @user = 
+      @user =
         case actor
         when User
           actor
@@ -125,7 +125,7 @@ module Gitlab
     def change_access_check(change)
       oldrev, newrev, ref = change.split(' ')
 
-      action = 
+      action =
         if project.protected_branch?(branch_name(ref))
           protected_branch_action(oldrev, newrev, branch_name(ref))
         elsif protected_tag?(tag_name(ref))
@@ -148,7 +148,7 @@ module Gitlab
             build_status_object(false, "You are not allowed to change existing tags on this project.")
           else # :push_code
             build_status_object(false, "You are not allowed to push code to this project.")
-          end 
+          end
         return status
       end
 
diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb
new file mode 100644
index 00000000000..4202c786466
--- /dev/null
+++ b/lib/gitlab/lfs/response.rb
@@ -0,0 +1,308 @@
+module Gitlab
+  module Lfs
+    class Response
+
+      def initialize(project, user, request)
+        @origin_project = project
+        @project = storage_project(project)
+        @user = user
+        @env = request.env
+        @request = request
+      end
+
+      # Return a response for a download request
+      # Can be a response to:
+      # Request from a user to get the file
+      # Request from gitlab-workhorse which file to serve to the user
+      def render_download_hypermedia_response(oid)
+        render_response_to_download do
+          if check_download_accept_header?
+            render_lfs_download_hypermedia(oid)
+          else
+            render_not_found
+          end
+        end
+      end
+
+      def render_download_object_response(oid)
+        render_response_to_download do
+          if check_download_sendfile_header? && check_download_accept_header?
+            render_lfs_sendfile(oid)
+          else
+            render_not_found
+          end
+        end
+      end
+
+      def render_lfs_api_auth
+        render_response_to_push do
+          request_body = JSON.parse(@request.body.read)
+          return render_not_found if request_body.empty? || request_body['objects'].empty?
+
+          response = build_response(request_body['objects'])
+          [
+            200,
+            {
+              "Content-Type" => "application/json; charset=utf-8",
+              "Cache-Control" => "private",
+            },
+            [JSON.dump(response)]
+          ]
+        end
+      end
+
+      def render_storage_upload_authorize_response(oid, size)
+        render_response_to_push do
+          [
+            200,
+            { "Content-Type" => "application/json; charset=utf-8" },
+            [JSON.dump({
+              'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload",
+              'LfsOid' => oid,
+              'LfsSize' => size
+            })]
+          ]
+        end
+      end
+
+      def render_storage_upload_store_response(oid, size, tmp_file_name)
+        render_response_to_push do
+          render_lfs_upload_ok(oid, size, tmp_file_name)
+        end
+      end
+
+      private
+
+      def render_not_enabled
+        [
+          501,
+          {
+            "Content-Type" => "application/vnd.git-lfs+json",
+          },
+          [JSON.dump({
+            'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.',
+            'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
+          })]
+        ]
+      end
+
+      def render_unauthorized
+        [
+          401,
+          {
+            'Content-Type' => 'text/plain'
+          },
+          ['Unauthorized']
+        ]
+      end
+
+      def render_not_found
+        [
+          404,
+          {
+            "Content-Type" => "application/vnd.git-lfs+json"
+          },
+          [JSON.dump({
+            'message' => 'Not found.',
+            'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
+          })]
+        ]
+      end
+
+      def render_forbidden
+        [
+          403,
+          {
+            "Content-Type" => "application/vnd.git-lfs+json"
+          },
+          [JSON.dump({
+            'message' => 'Access forbidden. Check your access level.',
+            'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
+          })]
+        ]
+      end
+
+      def render_lfs_sendfile(oid)
+        return render_not_found unless oid.present?
+
+        lfs_object = object_for_download(oid)
+
+        if lfs_object && lfs_object.file.exists?
+          [
+            200,
+            {
+              # GitLab-workhorse will forward Content-Type header
+              "Content-Type" => "application/octet-stream",
+              "X-Sendfile" => lfs_object.file.path
+            },
+            []
+          ]
+        else
+          render_not_found
+        end
+      end
+
+      def render_lfs_download_hypermedia(oid)
+        return render_not_found unless oid.present?
+
+        lfs_object = object_for_download(oid)
+        if lfs_object
+          [
+            200,
+            { "Content-Type" => "application/vnd.git-lfs+json" },
+            [JSON.dump(download_hypermedia(oid))]
+          ]
+        else
+          render_not_found
+        end
+      end
+
+      def render_lfs_upload_ok(oid, size, tmp_file)
+        if store_file(oid, size, tmp_file)
+          [
+            200,
+            {
+              'Content-Type' => 'text/plain',
+              'Content-Length' => 0
+            },
+            []
+          ]
+        else
+          [
+            422,
+            { 'Content-Type' => 'text/plain' },
+            ["Unprocessable entity"]
+          ]
+        end
+      end
+
+      def render_response_to_download
+        return render_not_enabled unless Gitlab.config.lfs.enabled
+
+        unless @project.public?
+          return render_unauthorized unless @user
+          return render_forbidden unless user_can_fetch?
+        end
+
+        yield
+      end
+
+      def render_response_to_push
+        return render_not_enabled unless Gitlab.config.lfs.enabled
+        return render_unauthorized unless @user
+        return render_forbidden unless user_can_push?
+
+        yield
+      end
+
+      def check_download_sendfile_header?
+        @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile"
+      end
+
+      def check_download_accept_header?
+        @env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8"
+      end
+
+      def user_can_fetch?
+        # Check user access against the project they used to initiate the pull
+        @user.can?(:download_code, @origin_project)
+      end
+
+      def user_can_push?
+        # Check user access against the project they used to initiate the push
+        @user.can?(:push_code, @origin_project)
+      end
+
+      def storage_project(project)
+        if project.forked?
+          project.forked_from_project
+        else
+          project
+        end
+      end
+
+      def store_file(oid, size, tmp_file)
+        tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file)
+
+        object = LfsObject.find_or_create_by(oid: oid, size: size)
+        if object.file.exists?
+          success = true
+        else
+          success = move_tmp_file_to_storage(object, tmp_file_path)
+        end
+
+        if success
+          success = link_to_project(object)
+        end
+
+        success
+      ensure
+        # Ensure that the tmp file is removed
+        FileUtils.rm_f(tmp_file_path)
+      end
+
+      def object_for_download(oid)
+        @project.lfs_objects.find_by(oid: oid)
+      end
+
+      def move_tmp_file_to_storage(object, path)
+        File.open(path) do |f|
+          object.file = f
+        end
+
+        object.file.store!
+        object.save
+      end
+
+      def link_to_project(object)
+        if object && !object.projects.exists?(@project)
+          object.projects << @project
+          object.save
+        end
+      end
+
+      def select_existing_objects(objects)
+        objects_oids = objects.map { |o| o['oid'] }
+        @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set
+      end
+
+      def build_response(objects)
+        selected_objects = select_existing_objects(objects)
+
+        upload_hypermedia(objects, selected_objects)
+      end
+
+      def download_hypermedia(oid)
+        {
+         '_links' => {
+           'download' =>
+             {
+              'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}",
+              'header' => {
+                'Accept' => "application/vnd.git-lfs+json; charset=utf-8",
+                'Authorization' => @env['HTTP_AUTHORIZATION']
+              }.compact
+            }
+          }
+        }
+      end
+
+      def upload_hypermedia(all_objects, existing_objects)
+        all_objects.each do |object|
+          object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid'])
+        end
+
+        { 'objects' => all_objects }
+      end
+
+      def hypermedia_links(object)
+        {
+          "upload" => {
+            'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}",
+            'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] }
+          }.compact
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb
new file mode 100644
index 00000000000..4809e834984
--- /dev/null
+++ b/lib/gitlab/lfs/router.rb
@@ -0,0 +1,95 @@
+module Gitlab
+  module Lfs
+    class Router
+      def initialize(project, user, request)
+        @project = project
+        @user = user
+        @env = request.env
+        @request = request
+      end
+
+      def try_call
+        return unless @request && @request.path.present?
+
+        case @request.request_method
+        when 'GET'
+          get_response
+        when 'POST'
+          post_response
+        when 'PUT'
+          put_response
+        else
+          nil
+        end
+      end
+
+      private
+
+      def get_response
+        path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/)
+        return nil unless path_match
+
+        oid = path_match[2]
+        return nil unless oid
+
+        case path_match[1]
+        when "info/lfs"
+          lfs.render_download_hypermedia_response(oid)
+        when "gitlab-lfs"
+          lfs.render_download_object_response(oid)
+        else
+          nil
+        end
+      end
+
+      def post_response
+        post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/)
+        return nil unless post_path
+
+        # Check for Batch API
+        if post_path[0].ends_with?("/info/lfs/objects/batch")
+          lfs.render_lfs_api_auth
+        else
+          nil
+        end
+      end
+
+      def put_response
+        object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/)
+        return nil if object_match.nil?
+
+        oid = object_match[1]
+        size = object_match[2].try(:to_i)
+        return nil if oid.nil? || size.nil?
+
+        # GitLab-workhorse requests
+        # 1. Try to authorize the request
+        # 2. send a request with a header containing the name of the temporary file
+        if object_match[3] && object_match[3] == '/authorize'
+          lfs.render_storage_upload_authorize_response(oid, size)
+        else
+          tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP'])
+          return nil unless tmp_file_name
+
+          lfs.render_storage_upload_store_response(oid, size, tmp_file_name)
+        end
+      end
+
+      def lfs
+        return unless @project
+
+        Gitlab::Lfs::Response.new(@project, @user, @request)
+      end
+
+      def sanitize_tmp_filename(name)
+        if name.present?
+          name.gsub!(/^.*(\\|\/)/, '')
+          name = name.match(/[0-9a-f]{73}/)
+          name[0] if name
+        else
+          nil
+        end
+      end
+    end
+  end
+end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index 0a7a4118077..93f2ad07aeb 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -44,7 +44,7 @@ upstream gitlab-workhorse {
 
 ## Normal HTTP host
 server {
-  ## Either remove "default_server" from the listen line below, 
+  ## Either remove "default_server" from the listen line below,
   ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab
   ## to be served if you visit any address that your server responds to, eg.
   ## the ip address of the server (http://x.x.x.x/)n 0.0.0.0:80 default_server;
@@ -113,6 +113,13 @@ server {
     proxy_pass http://gitlab;
   }
 
+  location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects {
+    client_max_body_size 0;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
   location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
     # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
     error_page 418 = @gitlab-workhorse;
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index b463d5b6aa9..90749947fa4 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -48,7 +48,7 @@ upstream gitlab-workhorse {
 
 ## Redirects all HTTP traffic to the HTTPS host
 server {
-  ## Either remove "default_server" from the listen line below, 
+  ## Either remove "default_server" from the listen line below,
   ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab
   ## to be served if you visit any address that your server responds to, eg.
   ## the ip address of the server (http://x.x.x.x/)
@@ -160,6 +160,13 @@ server {
     proxy_pass http://gitlab;
   }
 
+  location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects {
+    client_max_body_size 0;
+    # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+    error_page 418 = @gitlab-workhorse;
+    return 418;
+  }
+
   location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
     # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
     error_page 418 = @gitlab-workhorse;
-- 
cgit v1.2.1


From cd513034e66a3eccb9022968af3a09bc7203a9c9 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Sat, 14 Nov 2015 19:31:14 +0100
Subject: Set less strict flay option for now

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/flay.rake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
index 5efffc2cdac..dfb9df4772a 100644
--- a/lib/tasks/flay.rake
+++ b/lib/tasks/flay.rake
@@ -1,6 +1,6 @@
 desc 'Code duplication analyze via flay'
 task :flay do
-  output = %x(bundle exec flay app/ lib/gitlab/)
+  output = %x(bundle exec flay --mass 30 app/ lib/gitlab/)
 
   if output.include? "Similar code found"
     puts output
-- 
cgit v1.2.1


From 79fa6c646cecf887e6de8f2739d958ae49bb6eb3 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Mon, 16 Nov 2015 21:56:28 +0100
Subject: Remove duplication in reference filters

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/markdown/abstract_reference_filter.rb   | 100 +++++++++++++++++++++
 lib/gitlab/markdown/issue_reference_filter.rb      |  63 ++-----------
 .../markdown/merge_request_reference_filter.rb     |  61 ++-----------
 lib/gitlab/markdown/snippet_reference_filter.rb    |  61 ++-----------
 4 files changed, 119 insertions(+), 166 deletions(-)
 create mode 100644 lib/gitlab/markdown/abstract_reference_filter.rb

(limited to 'lib')

diff --git a/lib/gitlab/markdown/abstract_reference_filter.rb b/lib/gitlab/markdown/abstract_reference_filter.rb
new file mode 100644
index 00000000000..fd5b7eb9332
--- /dev/null
+++ b/lib/gitlab/markdown/abstract_reference_filter.rb
@@ -0,0 +1,100 @@
+require 'gitlab/markdown'
+
+module Gitlab
+  module Markdown
+    # Issues, Snippets and Merge Requests shares similar functionality in refernce filtering.
+    # All this functionality moved to this class
+    class AbstractReferenceFilter < ReferenceFilter
+      include CrossProjectReference
+
+      def self.object_class
+        # Implement in child class
+        # Example: MergeRequest
+      end
+
+      def self.object_name
+        object_class.name.underscore
+      end
+
+      def self.object_sym
+        object_name.to_sym
+      end
+
+      def self.data_reference
+        "data-#{object_name.dasherize}"
+      end
+
+      # Public: Find references in text (like `!123` for merge requests)
+      #
+      #   AnyReferenceFilter.references_in(text) do |match, object|
+      #     "<a href=...>PREFIX#{object}</a>"
+      #   end
+      #
+      # PREFIX - symbol that detects reference (like ! for merge requests)
+      # object - reference object (snippet, merget request etc)
+      # text - String text to search.
+      #
+      # Yields the String match, the Integer referenced object ID, and an optional String
+      # of the external project reference.
+      #
+      # Returns a String replaced with the return of the block.
+      def self.references_in(text)
+        text.gsub(object_class.reference_pattern) do |match|
+          yield match, $~[object_sym].to_i, $~[:project]
+        end
+      end
+
+      def self.referenced_by(node)
+        { object_sym => LazyReference.new(object_class, node.attr(data_reference)) }
+      end
+
+      delegate :object_class, :object_sym, :references_in, to: :class
+
+      def find_object(project, id)
+        # Implement in child class
+        # Example: project.merge_requests.find
+      end
+
+      def url_for_object(object, project)
+        # Implement in child class
+        # Example: project_merge_request_url
+      end
+
+      def call
+        replace_text_nodes_matching(object_class.reference_pattern) do |content|
+          object_link_filter(content)
+        end
+      end
+
+      # Replace references (like `!123` for merge requests) in text with links
+      # to the referenced object's details page.
+      #
+      # text - String text to replace references in.
+      #
+      # Returns a String with references replaced with links. All links
+      # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling.
+      def object_link_filter(text)
+        references_in(text) do |match, id, project_ref|
+          project = project_from_ref(project_ref)
+
+          if project && object = find_object(project, id)
+            title = escape_once("#{object_title}: #{object.title}")
+            klass = reference_class(object_sym)
+            data  = data_attribute(project: project.id, object_sym => object.id)
+            url = url_for_object(object, project)
+
+            %(<a href="#{url}" #{data}
+                 title="#{title}"
+                 class="#{klass}">#{match}</a>)
+          else
+            match
+          end
+        end
+      end
+
+      def object_title
+        object_class.name.titleize
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb
index 481d282f7b1..1ed69e2f431 100644
--- a/lib/gitlab/markdown/issue_reference_filter.rb
+++ b/lib/gitlab/markdown/issue_reference_filter.rb
@@ -6,66 +6,17 @@ module Gitlab
     # issues that do not exist are ignored.
     #
     # This filter supports cross-project references.
-    class IssueReferenceFilter < ReferenceFilter
-      include CrossProjectReference
-
-      # Public: Find `#123` issue references in text
-      #
-      #   IssueReferenceFilter.references_in(text) do |match, issue, project_ref|
-      #     "<a href=...>##{issue}</a>"
-      #   end
-      #
-      # text - String text to search.
-      #
-      # Yields the String match, the Integer issue ID, and an optional String of
-      # the external project reference.
-      #
-      # Returns a String replaced with the return of the block.
-      def self.references_in(text)
-        text.gsub(Issue.reference_pattern) do |match|
-          yield match, $~[:issue].to_i, $~[:project]
-        end
-      end
-
-      def self.referenced_by(node)
-        { issue: LazyReference.new(Issue, node.attr("data-issue")) }
-      end
-
-      def call
-        replace_text_nodes_matching(Issue.reference_pattern) do |content|
-          issue_link_filter(content)
-        end
+    class IssueReferenceFilter < AbstractReferenceFilter
+      def self.object_class
+        Issue
       end
 
-      # Replace `#123` issue references in text with links to the referenced
-      # issue's details page.
-      #
-      # text - String text to replace references in.
-      #
-      # Returns a String with `#123` references replaced with links. All links
-      # have `gfm` and `gfm-issue` class names attached for styling.
-      def issue_link_filter(text)
-        self.class.references_in(text) do |match, id, project_ref|
-          project = self.project_from_ref(project_ref)
-
-          if project && issue = project.get_issue(id)
-            url = url_for_issue(id, project, only_path: context[:only_path])
-
-            title = escape_once("Issue: #{issue.title}")
-            klass = reference_class(:issue)
-            data  = data_attribute(project: project.id, issue: issue.id)
-
-            %(<a href="#{url}" #{data}
-                 title="#{title}"
-                 class="#{klass}">#{match}</a>)
-          else
-            match
-          end
-        end
+      def find_object(project, id)
+        project.get_issue(id)
       end
 
-      def url_for_issue(*args)
-        IssuesHelper.url_for_issue(*args)
+      def url_for_object(issue, project)
+        IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path])
       end
     end
   end
diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb
index 5bc63269808..1f47f03c94e 100644
--- a/lib/gitlab/markdown/merge_request_reference_filter.rb
+++ b/lib/gitlab/markdown/merge_request_reference_filter.rb
@@ -6,65 +6,16 @@ module Gitlab
     # to merge requests that do not exist are ignored.
     #
     # This filter supports cross-project references.
-    class MergeRequestReferenceFilter < ReferenceFilter
-      include CrossProjectReference
-
-      # Public: Find `!123` merge request references in text
-      #
-      #   MergeRequestReferenceFilter.references_in(text) do |match, merge_request, project_ref|
-      #     "<a href=...>##{merge_request}</a>"
-      #   end
-      #
-      # text - String text to search.
-      #
-      # Yields the String match, the Integer merge request ID, and an optional
-      # String of the external project reference.
-      #
-      # Returns a String replaced with the return of the block.
-      def self.references_in(text)
-        text.gsub(MergeRequest.reference_pattern) do |match|
-          yield match, $~[:merge_request].to_i, $~[:project]
-        end
-      end
-
-      def self.referenced_by(node)
-        { merge_request: LazyReference.new(MergeRequest, node.attr("data-merge-request")) }
-      end
-
-      def call
-        replace_text_nodes_matching(MergeRequest.reference_pattern) do |content|
-          merge_request_link_filter(content)
-        end
+    class MergeRequestReferenceFilter < AbstractReferenceFilter
+      def self.object_class
+        MergeRequest
       end
 
-      # Replace `!123` merge request references in text with links to the
-      # referenced merge request's details page.
-      #
-      # text - String text to replace references in.
-      #
-      # Returns a String with `!123` references replaced with links. All links
-      # have `gfm` and `gfm-merge_request` class names attached for styling.
-      def merge_request_link_filter(text)
-        self.class.references_in(text) do |match, id, project_ref|
-          project = self.project_from_ref(project_ref)
-
-          if project && merge_request = project.merge_requests.find_by(iid: id)
-            title = escape_once("Merge Request: #{merge_request.title}")
-            klass = reference_class(:merge_request)
-            data  = data_attribute(project: project.id, merge_request: merge_request.id)
-
-            url = url_for_merge_request(merge_request, project)
-
-            %(<a href="#{url}" #{data}
-                 title="#{title}"
-                 class="#{klass}">#{match}</a>)
-          else
-            match
-          end
-        end
+      def find_object(project, id)
+        project.merge_requests.find_by(iid: id)
       end
 
-      def url_for_merge_request(mr, project)
+      def url_for_object(mr, project)
         h = Gitlab::Application.routes.url_helpers
         h.namespace_project_merge_request_url(project.namespace, project, mr,
                                             only_path: context[:only_path])
diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb
index f783f951711..f7bd07c2a34 100644
--- a/lib/gitlab/markdown/snippet_reference_filter.rb
+++ b/lib/gitlab/markdown/snippet_reference_filter.rb
@@ -6,65 +6,16 @@ module Gitlab
     # snippets that do not exist are ignored.
     #
     # This filter supports cross-project references.
-    class SnippetReferenceFilter < ReferenceFilter
-      include CrossProjectReference
-
-      # Public: Find `$123` snippet references in text
-      #
-      #   SnippetReferenceFilter.references_in(text) do |match, snippet|
-      #     "<a href=...>$#{snippet}</a>"
-      #   end
-      #
-      # text - String text to search.
-      #
-      # Yields the String match, the Integer snippet ID, and an optional String
-      # of the external project reference.
-      #
-      # Returns a String replaced with the return of the block.
-      def self.references_in(text)
-        text.gsub(Snippet.reference_pattern) do |match|
-          yield match, $~[:snippet].to_i, $~[:project]
-        end
-      end
-
-      def self.referenced_by(node)
-        { snippet: LazyReference.new(Snippet, node.attr("data-snippet")) }
-      end
-
-      def call
-        replace_text_nodes_matching(Snippet.reference_pattern) do |content|
-          snippet_link_filter(content)
-        end
+    class SnippetReferenceFilter < AbstractReferenceFilter
+      def self.object_class
+        Snippet
       end
 
-      # Replace `$123` snippet references in text with links to the referenced
-      # snippets's details page.
-      #
-      # text - String text to replace references in.
-      #
-      # Returns a String with `$123` references replaced with links. All links
-      # have `gfm` and `gfm-snippet` class names attached for styling.
-      def snippet_link_filter(text)
-        self.class.references_in(text) do |match, id, project_ref|
-          project = self.project_from_ref(project_ref)
-
-          if project && snippet = project.snippets.find_by(id: id)
-            title = escape_once("Snippet: #{snippet.title}")
-            klass = reference_class(:snippet)
-            data  = data_attribute(project: project.id, snippet: snippet.id)
-
-            url = url_for_snippet(snippet, project)
-
-            %(<a href="#{url}" #{data}
-                 title="#{title}"
-                 class="#{klass}">#{match}</a>)
-          else
-            match
-          end
-        end
+      def find_object(project, id)
+        project.snippets.find_by(id: id)
       end
 
-      def url_for_snippet(snippet, project)
+      def url_for_object(snippet, project)
         h = Gitlab::Application.routes.url_helpers
         h.namespace_project_snippet_url(project.namespace, project, snippet,
                                         only_path: context[:only_path])
-- 
cgit v1.2.1


From 4747412a49baf2d4776833e962e82b8cc893a06a Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Tue, 17 Nov 2015 11:18:01 +0100
Subject: Set higher flay value to avoid unnecessary refactoring for now

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/tasks/flay.rake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
index dfb9df4772a..e9587595fef 100644
--- a/lib/tasks/flay.rake
+++ b/lib/tasks/flay.rake
@@ -1,6 +1,6 @@
 desc 'Code duplication analyze via flay'
 task :flay do
-  output = %x(bundle exec flay --mass 30 app/ lib/gitlab/)
+  output = %x(bundle exec flay --mass 35 app/ lib/gitlab/)
 
   if output.include? "Similar code found"
     puts output
-- 
cgit v1.2.1


From 49e32cb59fbb5412630426152aa4a7f38f02a443 Mon Sep 17 00:00:00 2001
From: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
Date: Tue, 17 Nov 2015 12:55:16 +0100
Subject: Remove small code duplication in user_reference_filter.rb

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
---
 lib/gitlab/markdown/user_reference_filter.rb | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb
index 2a594e1662e..ab5e1f6fe9e 100644
--- a/lib/gitlab/markdown/user_reference_filter.rb
+++ b/lib/gitlab/markdown/user_reference_filter.rb
@@ -85,13 +85,12 @@ module Gitlab
 
       def link_to_all
         project = context[:project]
-
         url = urls.namespace_project_url(project.namespace, project,
                                          only_path: context[:only_path])
         data = data_attribute(project: project.id)
-
         text = User.reference_prefix + 'all'
-        %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>)
+
+        link_tag(url, data, text)
       end
 
       def link_to_namespace(namespace)
@@ -105,16 +104,20 @@ module Gitlab
       def link_to_group(group, namespace)
         url = urls.group_url(group, only_path: context[:only_path])
         data = data_attribute(group: namespace.id)
-
         text = Group.reference_prefix + group
-        %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>)
+
+        link_tag(url, data, text)
       end
 
       def link_to_user(user, namespace)
         url = urls.user_url(user, only_path: context[:only_path])
         data = data_attribute(user: namespace.owner_id)
-
         text = User.reference_prefix + user
+
+        link_tag(url, data, text)
+      end
+
+      def link_tag(url, data, text)
         %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>)
       end
     end
-- 
cgit v1.2.1