Merge remote-tracking branch 'origin/master' into repo-bugsrepo-bugs

22 files changed, 307 insertions, 59 deletions

diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index b87f7cdbad1..a989394ad91 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -130,7 +130,7 @@ module API
 
         commit = user_project.repository.commit(branch.dereferenced_target)
 
-        destroy_conditionally!(commit, last_update_field: :authored_date) do
+        destroy_conditionally!(commit, last_updated: commit.authored_date) do
           result = DeleteBranchService.new(user_project, current_user)
                     .execute(params[:branch])
 
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index e8dd61e493f..803b48dd88a 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -320,7 +320,10 @@ module API
     end
 
     class IssueBasic < ProjectEntity
-      expose :label_names, as: :labels
+      expose :labels do |issue, options|
+        # Avoids an N+1 query since labels are preloaded
+        issue.labels.map(&:title).sort
+      end
       expose :milestone, using: Entities::Milestone
       expose :assignees, :author, using: Entities::UserBasic
 
@@ -329,13 +332,32 @@ module API
       end
 
       expose :user_notes_count
-      expose :upvotes, :downvotes
+      expose :upvotes do |issue, options|
+        if options[:issuable_metadata]
+          # Avoids an N+1 query when metadata is included
+          options[:issuable_metadata][issue.id].upvotes
+        else
+          issue.upvotes
+        end
+      end
+      expose :downvotes do |issue, options|
+        if options[:issuable_metadata]
+          # Avoids an N+1 query when metadata is included
+          options[:issuable_metadata][issue.id].downvotes
+        else
+          issue.downvotes
+        end
+      end
       expose :due_date
       expose :confidential
 
       expose :web_url do |issue, options|
         Gitlab::UrlBuilder.build(issue)
       end
+
+      expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |issue|
+        issue
+      end
     end
 
     class Issue < IssueBasic
@@ -365,10 +387,22 @@ module API
     end
 
     class IssuableTimeStats < Grape::Entity
+      format_with(:time_tracking_formatter) do |time_spent|
+        Gitlab::TimeTrackingFormatter.output(time_spent)
+      end
+
       expose :time_estimate
       expose :total_time_spent
       expose :human_time_estimate
-      expose :human_total_time_spent
+
+      with_options(format_with: :time_tracking_formatter) do
+        expose :total_time_spent, as: :human_total_time_spent
+      end
+
+      def total_time_spent
+        # Avoids an N+1 query since timelogs are preloaded
+        object.timelogs.map(&:time_spent).sum
+      end
     end
 
     class ExternalIssue < Grape::Entity
@@ -418,6 +452,10 @@ module API
       expose :web_url do |merge_request, options|
         Gitlab::UrlBuilder.build(merge_request)
       end
+
+      expose :time_stats, using: 'API::Entities::IssuableTimeStats' do |merge_request|
+        merge_request
+      end
     end
 
     class MergeRequest < MergeRequestBasic
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index ee2ad27837b..8c494a54329 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -7,7 +7,11 @@ module API
     helpers do
       params :optional_params_ce do
         optional :description, type: String, desc: 'The description of the group'
-        optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group'
+        optional :visibility, type: String,
+                              values: Gitlab::VisibilityLevel.string_values,
+                              default: Gitlab::VisibilityLevel.string_level(
+                                Gitlab::CurrentSettings.current_application_settings.default_group_visibility),
+                              desc: 'The visibility of the group'
         optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
         optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group'
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 84980864151..3d377fdb9eb 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -19,8 +19,10 @@ module API
       end
     end
 
-    def destroy_conditionally!(resource, last_update_field: :updated_at)
-      check_unmodified_since!(resource.public_send(last_update_field))
+    def destroy_conditionally!(resource, last_updated: nil)
+      last_updated ||= resource.updated_at
+
+      check_unmodified_since!(last_updated)
 
       status 204
       if block_given?
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 6503629e2a2..0297023226f 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -4,6 +4,8 @@ module API
 
     before { authenticate! }
 
+    helpers ::Gitlab::IssuableMetadata
+
     helpers do
       def find_issues(args = {})
         args = params.merge(args)
@@ -13,6 +15,7 @@ module API
         args[:label_name] = args.delete(:labels)
 
         issues = IssuesFinder.new(current_user, args).execute
+          .preload(:assignees, :labels, :notes, :timelogs)
 
         issues.reorder(args[:order_by] => args[:sort])
       end
@@ -65,7 +68,13 @@ module API
       get do
         issues = find_issues
 
-        present paginate(issues), with: Entities::IssueBasic, current_user: current_user
+        options = {
+          with: Entities::IssueBasic,
+          current_user: current_user,
+          issuable_metadata: issuable_meta_data(issues, 'Issue')
+        }
+
+        present paginate(issues), options
       end
     end
 
@@ -86,7 +95,13 @@ module API
 
         issues = find_issues(group_id: group.id)
 
-        present paginate(issues), with: Entities::IssueBasic, current_user: current_user
+        options = {
+          with: Entities::IssueBasic,
+          current_user: current_user,
+          issuable_metadata: issuable_meta_data(issues, 'Issue')
+        }
+
+        present paginate(issues), options
       end
     end
 
@@ -109,7 +124,14 @@ module API
 
         issues = find_issues(project_id: project.id)
 
-        present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project
+        options = {
+          with: Entities::IssueBasic,
+          current_user: current_user,
+          project: user_project,
+          issuable_metadata: issuable_meta_data(issues, 'Issue')
+        }
+
+        present paginate(issues), options
       end
 
       desc 'Get a single project issue' do
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 969c6064662..eec8d9357aa 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -21,7 +21,7 @@ module API
         return merge_requests if args[:view] == 'simple'
 
         merge_requests
-          .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels)
+          .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels, :timelogs)
       end
 
       params :merge_requests_params do
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 81b17935b81..912415e3a7f 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -70,7 +70,7 @@ module API
 
         commit = user_project.repository.commit(tag.dereferenced_target)
 
-        destroy_conditionally!(commit, last_update_field: :authored_date) do
+        destroy_conditionally!(commit, last_updated: commit.authored_date) do
           result = ::Tags::DestroyService.new(user_project, current_user)
                     .execute(params[:tag_name])
 
diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb
index 1c8b55ee4c4..3e87a09704e 100644
--- a/lib/gitlab/ci/config/entry/attributable.rb
+++ b/lib/gitlab/ci/config/entry/attributable.rb
@@ -8,6 +8,10 @@ module Gitlab
           class_methods do
             def attributes(*attributes)
               attributes.flatten.each do |attribute|
+                if method_defined?(attribute)
+                  raise ArgumentError, 'Method already defined!'
+                end
+
                 define_method(attribute) do
                   return unless config.is_a?(Hash)
 
diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb
index e05aca9881b..68b6742385a 100644
--- a/lib/gitlab/ci/config/entry/configurable.rb
+++ b/lib/gitlab/ci/config/entry/configurable.rb
@@ -15,9 +15,10 @@ module Gitlab
         #
         module Configurable
           extend ActiveSupport::Concern
-          include Validatable
 
           included do
+            include Validatable
+
             validations do
               validates :config, type: Hash
             end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 32f5c6ab142..91aac6df4b1 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -59,10 +59,10 @@ module Gitlab
           entry :services, Entry::Services,
             description: 'Services that will be used to execute this job.'
 
-          entry :only, Entry::Trigger,
+          entry :only, Entry::Policy,
             description: 'Refs policy this job will be executed for.'
 
-          entry :except, Entry::Trigger,
+          entry :except, Entry::Policy,
             description: 'Refs policy this job will be executed for.'
 
           entry :variables, Entry::Variables,
diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb
index a6a914d79c1..c868943c42e 100644
--- a/lib/gitlab/ci/config/entry/node.rb
+++ b/lib/gitlab/ci/config/entry/node.rb
@@ -16,8 +16,9 @@ module Gitlab
             @metadata = metadata
             @entries = {}
 
-            @validator = self.class.validator.new(self)
-            @validator.validate(:new)
+            self.class.aspects.to_a.each do |aspect|
+              instance_exec(&aspect)
+            end
           end
 
           def [](key)
@@ -47,7 +48,7 @@ module Gitlab
           end
 
           def errors
-            @validator.messages + descendants.flat_map(&:errors)
+            []
           end
 
           def value
@@ -70,6 +71,13 @@ module Gitlab
             true
           end
 
+          def location
+            name = @key.presence || self.class.name.to_s.demodulize
+                                        .underscore.humanize.downcase
+
+            ancestors.map(&:key).append(name).compact.join(':')
+          end
+
           def inspect
             val = leaf? ? config : descendants
             unspecified = specified? ? '' : '(unspecified) '
@@ -79,8 +87,8 @@ module Gitlab
           def self.default
           end
 
-          def self.validator
-            Validator
+          def self.aspects
+            @aspects ||= []
           end
         end
       end
diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb
new file mode 100644
index 00000000000..3cdae1cee4f
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/policy.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents an only/except trigger policy for the job.
+        #
+        class Policy < Simplifiable
+          strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) }
+
+          class RefsPolicy < Entry::Node
+            include Entry::Validatable
+
+            validations do
+              validates :config, array_of_strings_or_regexps: true
+            end
+          end
+
+          class UnknownStrategy < Entry::Node
+            def errors
+              ["#{location} has to be either an array of conditions or a hash"]
+            end
+          end
+
+          def self.default
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb
new file mode 100644
index 00000000000..12764629686
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/simplifiable.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Simplifiable < SimpleDelegator
+          EntryStrategy = Struct.new(:name, :condition)
+
+          def initialize(config, **metadata)
+            unless self.class.const_defined?(:UnknownStrategy)
+              raise ArgumentError, 'UndefinedStrategy not available!'
+            end
+
+            strategy = self.class.strategies.find do |variant|
+              variant.condition.call(config)
+            end
+
+            entry = self.class.entry_class(strategy)
+
+            super(entry.new(config, metadata))
+          end
+
+          def self.strategy(name, **opts)
+            EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy|
+              strategies.append(strategy)
+            end
+          end
+
+          def self.strategies
+            @strategies ||= []
+          end
+
+          def self.entry_class(strategy)
+            if strategy.present?
+              self.const_get(strategy.name)
+            else
+              self::UnknownStrategy
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb
deleted file mode 100644
index 16b234e6c59..00000000000
--- a/lib/gitlab/ci/config/entry/trigger.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # Entry that represents a trigger policy for the job.
-        #
-        class Trigger < Node
-          include Validatable
-
-          validations do
-            validates :config, array_of_strings_or_regexps: true
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb
index f7f1b111571..5ced778d311 100644
--- a/lib/gitlab/ci/config/entry/validatable.rb
+++ b/lib/gitlab/ci/config/entry/validatable.rb
@@ -5,6 +5,17 @@ module Gitlab
         module Validatable
           extend ActiveSupport::Concern
 
+          def self.included(node)
+            node.aspects.append -> do
+              @validator = self.class.validator.new(self)
+              @validator.validate(:new)
+            end
+          end
+
+          def errors
+            @validator.messages + descendants.flat_map(&:errors)
+          end
+
           class_methods do
             def validator
               @validator ||= Class.new(Entry::Validator).tap do |validator|
diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb
index 55343005fe3..2df23a3edcd 100644
--- a/lib/gitlab/ci/config/entry/validator.rb
+++ b/lib/gitlab/ci/config/entry/validator.rb
@@ -8,7 +8,6 @@ module Gitlab
 
           def initialize(entry)
             super(entry)
-            @entry = entry
           end
 
           def messages
@@ -20,21 +19,6 @@ module Gitlab
           def self.name
             'Validator'
           end
-
-          private
-
-          def location
-            predecessors = ancestors.map(&:key).compact
-            predecessors.append(key_name).join(':')
-          end
-
-          def key_name
-            if key.blank?
-              @entry.class.name.demodulize.underscore.humanize
-            else
-              key
-            end
-          end
         end
       end
     end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index e001d25e7b7..a6ec75da385 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -9,6 +9,14 @@ module Gitlab
       ActiveRecord::Base.configurations[Rails.env]
     end
 
+    def self.username
+      config['username'] || ENV['USER']
+    end
+
+    def self.database_name
+      config['database']
+    end
+
     def self.adapter_name
       config['adapter']
     end
diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb
new file mode 100644
index 00000000000..aee3981e79a
--- /dev/null
+++ b/lib/gitlab/database/grant.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Database
+    # Model that can be used for querying permissions of a SQL user.
+    class Grant < ActiveRecord::Base
+      self.table_name =
+        if Database.postgresql?
+          'information_schema.role_table_grants'
+        else
+          'mysql.user'
+        end
+
+      def self.scope_to_current_user
+        if Database.postgresql?
+          where('grantee = user')
+        else
+          where("CONCAT(User, '@', Host) = current_user()")
+        end
+      end
+
+      # Returns true if the current user can create and execute triggers on the
+      # given table.
+      def self.create_and_execute_trigger?(table)
+        priv =
+          if Database.postgresql?
+            where(privilege_type: 'TRIGGER', table_name: table)
+          else
+            where(Trigger_priv: 'Y')
+          end
+
+        priv.scope_to_current_user.any?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 5e2c6cc5cad..fb14798efe6 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -358,6 +358,8 @@ module Gitlab
           raise 'rename_column_concurrently can not be run inside a transaction'
         end
 
+        check_trigger_permissions!(table)
+
         old_col = column_for(table, old)
         new_type = type || old_col.type
 
@@ -430,6 +432,8 @@ module Gitlab
       def cleanup_concurrent_column_rename(table, old, new)
         trigger_name = rename_trigger_name(table, old, new)
 
+        check_trigger_permissions!(table)
+
         if Database.postgresql?
           remove_rename_triggers_for_postgresql(table, trigger_name)
         else
@@ -485,14 +489,14 @@ module Gitlab
 
       # Removes the triggers used for renaming a PostgreSQL column concurrently.
       def remove_rename_triggers_for_postgresql(table, trigger)
-        execute("DROP TRIGGER #{trigger} ON #{table}")
-        execute("DROP FUNCTION #{trigger}()")
+        execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}")
+        execute("DROP FUNCTION IF EXISTS #{trigger}()")
       end
 
       # Removes the triggers used for renaming a MySQL column concurrently.
       def remove_rename_triggers_for_mysql(trigger)
-        execute("DROP TRIGGER #{trigger}_insert")
-        execute("DROP TRIGGER #{trigger}_update")
+        execute("DROP TRIGGER IF EXISTS #{trigger}_insert")
+        execute("DROP TRIGGER IF EXISTS #{trigger}_update")
       end
 
       # Returns the (base) name to use for triggers when renaming columns.
@@ -625,6 +629,30 @@ module Gitlab
           conn.llen("queue:#{queue_name}")
         end
       end
+
+      def check_trigger_permissions!(table)
+        unless Grant.create_and_execute_trigger?(table)
+          dbname = Database.database_name
+          user = Database.username
+
+          raise <<-EOF
+Your database user is not allowed to create, drop, or execute triggers on the
+table #{table}.
+
+If you are using PostgreSQL you can solve this by logging in to the GitLab
+database (#{dbname}) using a super user and running:
+
+    ALTER #{user} WITH SUPERUSER
+
+For MySQL you instead need to run:
+
+    GRANT ALL PRIVILEGES ON *.* TO #{user}@'%'
+
+Both queries will grant the user super user permissions, ensuring you don't run
+into similar problems in the future (e.g. when new tables are created).
+          EOF
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 03e2bec84dd..fb6504bdea0 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -17,6 +17,7 @@ module Gitlab
       NoRepository = Class.new(StandardError)
       InvalidBlobName = Class.new(StandardError)
       InvalidRef = Class.new(StandardError)
+      GitError = Class.new(StandardError)
 
       class << self
         # Unlike `new`, `create` takes the storage path, not the storage name
@@ -246,6 +247,13 @@ module Gitlab
         branch_names + tag_names
       end
 
+      # Returns an Array of all ref names, except when it's matching pattern
+      #
+      # regexp - The pattern for ref names we don't want
+      def all_ref_names_except(regexp)
+        rugged.references.reject { |ref| ref.name =~ regexp }.map(&:name)
+      end
+
       # Discovers the default branch based on the repository's available branches
       #
       # - If no branches are present, returns nil
@@ -591,6 +599,23 @@ module Gitlab
         rugged.branches.delete(branch_name)
       end
 
+      def delete_refs(*ref_names)
+        instructions = ref_names.map do |ref|
+          "delete #{ref}\x00\x00"
+        end
+
+        command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z]
+        message, status = Gitlab::Popen.popen(
+          command,
+          path) do |stdin|
+          stdin.write(instructions.join)
+        end
+
+        unless status.zero?
+          raise GitError.new("Could not delete refs #{ref_names}: #{message}")
+        end
+      end
+
       # Create a new branch named **ref+ based on **stat_point+, HEAD by default
       #
       # Examples:
diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb
new file mode 100644
index 00000000000..b42bc67ccfc
--- /dev/null
+++ b/lib/gitlab/sql/pattern.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module SQL
+    module Pattern
+      extend ActiveSupport::Concern
+
+      MIN_CHARS_FOR_PARTIAL_MATCHING = 3
+
+      class_methods do
+        def to_pattern(query)
+          if partial_matching?(query)
+            "%#{sanitize_sql_like(query)}%"
+          else
+            sanitize_sql_like(query)
+          end
+        end
+
+        def partial_matching?(query)
+          query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index f76bef5f4bf..8ae1b6a626a 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -111,7 +111,7 @@ namespace :gitlab do
           next unless id > max_iid
 
           project.deployments.find(id).create_ref
-          rugged.references.delete(ref)
+          project.repository.delete_refs(ref)
         end
       end
     end