Merge branch 'master' into karlhungus-mr-on-fork

Conflicts:
	app/contexts/filter_context.rb
	app/contexts/search_context.rb
	app/models/merge_request.rb
	app/models/note.rb
	app/views/shared/_merge_requests.html.haml
	spec/controllers/commit_controller_spec.rb
	spec/services/notification_service_spec.rb

10 files changed, 221 insertions, 105 deletions

diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index a602dc05418..79f8eb3a543 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -1,6 +1,10 @@
 module API
   # Internal access API
   class Internal < Grape::API
+
+    DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }
+    PUSH_COMMANDS = %w{ git-receive-pack }
+
     namespace 'internal' do
       #
       # Check if ssh key has access to project code
@@ -26,16 +30,16 @@ module API
 
 
         if key.is_a? DeployKey
-          key.projects.include?(project) && git_cmd == 'git-upload-pack'
+          key.projects.include?(project) && DOWNLOAD_COMMANDS.include?(git_cmd)
         else
           user = key.user
 
           return false if user.blocked?
 
           action = case git_cmd
-                   when 'git-upload-pack', 'git-upload-archive'
+                   when *DOWNLOAD_COMMANDS
                      then :download_code
-                   when 'git-receive-pack'
+                   when *PUSH_COMMANDS
                      then
                      if project.protected_branch?(params[:ref])
                        :push_code_to_protected_branches
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
new file mode 100644
index 00000000000..258a0fb2589
--- /dev/null
+++ b/lib/backup/manager.rb
@@ -0,0 +1,106 @@
+module Backup
+  class Manager
+    def pack
+      # saving additional informations
+      s = {}
+      s[:db_version]         = "#{ActiveRecord::Migrator.current_version}"
+      s[:backup_created_at]  = Time.now
+      s[:gitlab_version]     = %x{git rev-parse HEAD}.gsub(/\n/,"")
+      s[:tar_version]        = %x{tar --version | head -1}.gsub(/\n/,"")
+
+      Dir.chdir(Gitlab.config.backup.path)
+
+      File.open("#{Gitlab.config.backup.path}/backup_information.yml", "w+") do |file|
+        file << s.to_yaml.gsub(/^---\n/,'')
+      end
+
+      # create archive
+      print "Creating backup archive: #{s[:backup_created_at].to_i}_gitlab_backup.tar ... "
+      if Kernel.system("tar -cf #{s[:backup_created_at].to_i}_gitlab_backup.tar repositories/ db/ uploads/ backup_information.yml")
+        puts "done".green
+      else
+        puts "failed".red
+      end
+    end
+
+    def cleanup
+      print "Deleting tmp directories ... "
+      if Kernel.system("rm -rf repositories/ db/ uploads/ backup_information.yml")
+        puts "done".green
+      else
+        puts "failed".red
+      end
+    end
+
+    def remove_old
+      # delete backups
+      print "Deleting old backups ... "
+      keep_time = Gitlab.config.backup.keep_time.to_i
+      path = Gitlab.config.backup.path
+
+      if keep_time > 0
+        removed = 0
+        file_list = Dir.glob(Rails.root.join(path, "*_gitlab_backup.tar"))
+        file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ }
+        file_list.sort.each do |timestamp|
+          if Time.at(timestamp) < (Time.now - keep_time)
+            if system("rm #{timestamp}_gitlab_backup.tar")
+              removed += 1
+            end
+          end
+        end
+        puts "done. (#{removed} removed)".green
+      else
+        puts "skipping".yellow
+      end
+    end
+
+    def unpack
+      Dir.chdir(Gitlab.config.backup.path)
+
+      # check for existing backups in the backup dir
+      file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i }
+      puts "no backups found" if file_list.count == 0
+      if file_list.count > 1 && ENV["BACKUP"].nil?
+        puts "Found more than one backup, please specify which one you want to restore:"
+        puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+        exit 1
+      end
+
+      tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar")
+
+      unless File.exists?(tar_file)
+        puts "The specified backup doesn't exist!"
+        exit 1
+      end
+
+      print "Unpacking backup ... "
+      unless Kernel.system("tar -xf #{tar_file}")
+        puts "failed".red
+        exit 1
+      else
+        puts "done".green
+      end
+
+      settings = YAML.load_file("backup_information.yml")
+      ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+      # backups directory is not always sub of Rails root and able to execute the git rev-parse below
+      begin
+        Dir.chdir(Rails.root)
+
+        # restoring mismatching backups can lead to unexpected problems
+        if settings[:gitlab_version] != %x{git rev-parse HEAD}.gsub(/\n/, "")
+          puts "GitLab version mismatch:".red
+          puts "  Your current HEAD differs from the HEAD in the backup!".red
+          puts "  Please switch to the following revision and try again:".red
+          puts "  revision: #{settings[:gitlab_version]}".red
+          exit 1
+        end
+      ensure
+        # chdir back to original intended dir
+        Dir.chdir(Gitlab.config.backup.path)
+      end
+    end
+  end
+end
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index a81c80cfc6f..d1035240cb6 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -95,13 +95,9 @@ module ExtractsPath
   # resolved (e.g., when a user inserts an invalid path or ref).
   def assign_ref_vars
     @id = get_id
-
     @ref, @path = extract_ref(@id)
-
     @repo = @project.repository
-
     @commit = @repo.commit(@ref)
-
     @tree = Tree.new(@repo, @commit.id, @ref, @path)
     @hex_path = Digest::SHA1.hexdigest(@path)
     @logs_path = logs_file_project_ref_path(@project, @ref, @path)
diff --git a/lib/gitlab/blacklist.rb b/lib/gitlab/blacklist.rb
index 59203b2fbd6..d00214a62cb 100644
--- a/lib/gitlab/blacklist.rb
+++ b/lib/gitlab/blacklist.rb
@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     def path
-      %w(admin dashboard groups help profile projects search public assets u s teams merge_requests issues users snippets services)
+      %w(admin dashboard groups help profile projects search public assets u s teams merge_requests issues users snippets services repository)
     end
   end
 end
diff --git a/lib/gitlab/diff_parser.rb b/lib/gitlab/diff_parser.rb
new file mode 100644
index 00000000000..fb27280c4a4
--- /dev/null
+++ b/lib/gitlab/diff_parser.rb
@@ -0,0 +1,77 @@
+module Gitlab
+  class DiffParser
+    include Enumerable
+
+    attr_reader :lines, :new_path
+
+    def initialize(diff)
+      @lines = diff.diff.lines.to_a
+      @new_path = diff.new_path
+    end
+
+    def each
+      line_old = 1
+      line_new = 1
+      type = nil
+
+      lines_arr = ::Gitlab::InlineDiff.processing lines
+      lines_arr.each do |line|
+        raw_line = line.dup
+
+        next if line.match(/^\-\-\- \/dev\/null/)
+        next if line.match(/^\+\+\+ \/dev\/null/)
+        next if line.match(/^\-\-\- a/)
+        next if line.match(/^\+\+\+ b/)
+
+        full_line = html_escape(line.gsub(/\n/, ''))
+        full_line = ::Gitlab::InlineDiff.replace_markers full_line
+
+        if line.match(/^@@ -/)
+          type = "match"
+
+          line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0
+          line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0
+
+          next if line_old == 1 && line_new == 1 #top of file
+          yield(full_line, type, nil, nil, nil)
+          next
+        else
+          type = identification_type(line)
+          line_code = generate_line_code(new_path, line_new, line_old)
+          yield(full_line, type, line_code, line_new, line_old, raw_line)
+        end
+
+
+        if line[0] == "+"
+          line_new += 1
+        elsif line[0] == "-"
+          line_old += 1
+        else
+          line_new += 1
+          line_old += 1
+        end
+      end
+    end
+
+    private
+
+    def identification_type(line)
+      if line[0] == "+"
+        "new"
+      elsif line[0] == "-"
+        "old"
+      else
+        nil
+      end
+    end
+
+    def generate_line_code(path, line_new, line_old)
+      "#{Digest::SHA1.hexdigest(path)}_#{line_old}_#{line_new}"
+    end
+
+    def html_escape str
+      replacements = { '&' => '&amp;', '>' => '&gt;', '<' => '&lt;', '"' => '&quot;', "'" => '&#39;' }
+        str.gsub(/[&"'><]/, replacements)
+    end
+  end
+end
diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb
index e9c4df220c9..95bb22cfc27 100644
--- a/lib/gitlab/markdown.rb
+++ b/lib/gitlab/markdown.rb
@@ -167,7 +167,7 @@ module Gitlab
 
     def reference_user(identifier)
       if member = @project.team_members.find { |user| user.username == identifier }
-        link_to("@#{identifier}", user_path(identifier), html_options.merge(class: "gfm gfm-team_member #{html_options[:class]}")) if member
+        link_to("@#{identifier}", user_url(identifier), html_options.merge(class: "gfm gfm-team_member #{html_options[:class]}")) if member
       end
     end
 
diff --git a/lib/gitlab/satellite/edit_file_action.rb b/lib/gitlab/satellite/edit_file_action.rb
index 07570965b1a..d793d0ba8dc 100644
--- a/lib/gitlab/satellite/edit_file_action.rb
+++ b/lib/gitlab/satellite/edit_file_action.rb
@@ -49,7 +49,7 @@ module Gitlab
       protected
 
       def can_edit?(last_commit)
-        current_last_commit = @project.repository.last_commit_for(ref, file_path).sha
+        current_last_commit = Gitlab::Git::Commit.last_for_path(@project.repository, ref, file_path).sha
         last_commit == current_last_commit
       end
     end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index d071938acb5..2eff1260b61 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -11,49 +11,10 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:create"].invoke
       Rake::Task["gitlab:backup:uploads:create"].invoke
 
-
-      # saving additional informations
-      s = {}
-      s[:db_version]         = "#{ActiveRecord::Migrator.current_version}"
-      s[:backup_created_at]  = Time.now
-      s[:gitlab_version]     = %x{git rev-parse HEAD}.gsub(/\n/,"")
-      s[:tar_version]        = %x{tar --version | head -1}.gsub(/\n/,"")
-
-      Dir.chdir(Gitlab.config.backup.path)
-
-      File.open("#{Gitlab.config.backup.path}/backup_information.yml", "w+") do |file|
-        file << s.to_yaml.gsub(/^---\n/,'')
-      end
-
-      # create archive
-      print "Creating backup archive: #{s[:backup_created_at].to_i}_gitlab_backup.tar ... "
-      if Kernel.system("tar -cf #{s[:backup_created_at].to_i}_gitlab_backup.tar repositories/ db/ uploads/ backup_information.yml")
-        puts "done".green
-      else
-        puts "failed".red
-      end
-
-      # cleanup: remove tmp files
-      print "Deleting tmp directories ... "
-      if Kernel.system("rm -rf repositories/ db/ uploads/ backup_information.yml")
-        puts "done".green
-      else
-        puts "failed".red
-      end
-
-      # delete backups
-      print "Deleting old backups ... "
-      if Gitlab.config.backup.keep_time > 0
-        file_list = Dir.glob("*_gitlab_backup.tar").map { |f| f.split(/_/).first.to_i }
-        file_list.sort.each do |timestamp|
-          if Time.at(timestamp) < (Time.now - Gitlab.config.backup.keep_time)
-            %x{rm #{timestamp}_gitlab_backup.tar}
-          end
-        end
-        puts "done".green
-      else
-        puts "skipping".yellow
-      end
+      backup = Backup::Manager.new
+      backup.pack
+      backup.cleanup
+      backup.remove_old
     end
 
     # Restore backup of GitLab system
@@ -61,64 +22,15 @@ namespace :gitlab do
     task restore: :environment do
       warn_user_is_not_gitlab
 
-      Dir.chdir(Gitlab.config.backup.path)
-
-      # check for existing backups in the backup dir
-      file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i }
-      puts "no backups found" if file_list.count == 0
-      if file_list.count > 1 && ENV["BACKUP"].nil?
-        puts "Found more than one backup, please specify which one you want to restore:"
-        puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
-        exit 1
-      end
-
-      tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar")
-
-      unless File.exists?(tar_file)
-        puts "The specified backup doesn't exist!"
-        exit 1
-      end
-
-      print "Unpacking backup ... "
-      unless Kernel.system("tar -xf #{tar_file}")
-        puts "failed".red
-        exit 1
-      else
-        puts "done".green
-      end
-
-      settings = YAML.load_file("backup_information.yml")
-      ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
-
-      # backups directory is not always sub of Rails root and able to execute the git rev-parse below
-      begin
-        Dir.chdir(Rails.root)
-
-        # restoring mismatching backups can lead to unexpected problems
-        if settings[:gitlab_version] != %x{git rev-parse HEAD}.gsub(/\n/, "")
-          puts "GitLab version mismatch:".red
-          puts "  Your current HEAD differs from the HEAD in the backup!".red
-          puts "  Please switch to the following revision and try again:".red
-          puts "  revision: #{settings[:gitlab_version]}".red
-          exit 1
-        end
-      ensure
-        # chdir back to original intended dir
-        Dir.chdir(Gitlab.config.backup.path)
-      end
+      backup = Backup::Manager.new
+      backup.unpack
 
       Rake::Task["gitlab:backup:db:restore"].invoke
       Rake::Task["gitlab:backup:repo:restore"].invoke
       Rake::Task["gitlab:backup:uploads:restore"].invoke
       Rake::Task["gitlab:shell:setup"].invoke
 
-      # cleanup: remove tmp files
-      print "Deleting tmp directories ... "
-      if Kernel.system("rm -rf repositories/ db/ uploads/ backup_information.yml")
-        puts "done".green
-      else
-        puts "failed".red
-      end
+      backup.cleanup
     end
 
     namespace :repo do
diff --git a/lib/tasks/migrate/migrate_inline_notes.rake b/lib/tasks/migrate/migrate_inline_notes.rake
new file mode 100644
index 00000000000..e21fa0e8ce8
--- /dev/null
+++ b/lib/tasks/migrate/migrate_inline_notes.rake
@@ -0,0 +1,16 @@
+desc "GITLAB | Migrate inline notes"
+task migrate_inline_notes: :environment do
+  Note.where('line_code IS NOT NULL').find_each(batch_size: 100) do |note|
+    begin
+      note.set_diff
+      if note.save
+        print '.'
+      else
+        print 'F'
+      end
+    rescue
+      print 'F'
+    end
+  end
+end
+
diff --git a/lib/tasks/migrate/migrate_keys.rake b/lib/tasks/migrate/migrate_keys.rake
index b13d984907b..969ba6dc15e 100644
--- a/lib/tasks/migrate/migrate_keys.rake
+++ b/lib/tasks/migrate/migrate_keys.rake
@@ -1,15 +1,20 @@
 desc "GITLAB | Migrate SSH Keys"
 task migrate_keys: :environment do
   puts "This will add fingerprint to ssh keys in db"
+  puts "If you have duplicate keys https://github.com/gitlabhq/gitlabhq/issues/4453 all but the first will be deleted".yellow
   ask_to_continue
 
   Key.find_each(batch_size: 20) do |key|
     if key.valid? && key.save
       print '.'
+    elsif key.fingerprint.present?
+      puts "\nDeleting #{key.inspect}".yellow
+      key.destroy
     else
       print 'F'
     end
   end
+  print "\n"
 end