Merge branch 'master' into experimental-breadcrumbsexperimental-breadcrumbs

30 files changed, 1524 insertions, 76 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index aa91451c9f4..cef5a0abe12 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -444,7 +444,15 @@ module API
     end
 
     class Namespace < Grape::Entity
-      expose :id, :name, :path, :kind, :full_path
+      expose :id, :name, :path, :kind, :full_path, :parent_id
+
+      expose :members_count_with_descendants, if: -> (namespace, opts) { expose_members_count_with_descendants?(namespace, opts) } do |namespace, _|
+        namespace.users_with_descendants.count
+      end
+
+      def expose_members_count_with_descendants?(namespace, opts)
+        namespace.kind == 'group' && Ability.allowed?(opts[:current_user], :admin_group, namespace)
+      end
     end
 
     class MemberAccess < Grape::Entity
diff --git a/lib/api/features.rb b/lib/api/features.rb
index cff0ba2ddff..21745916463 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -2,6 +2,29 @@ module API
   class Features < Grape::API
     before { authenticated_as_admin! }
 
+    helpers do
+      def gate_value(params)
+        case params[:value]
+        when 'true'
+          true
+        when '0', 'false'
+          false
+        else
+          params[:value].to_i
+        end
+      end
+
+      def gate_target(params)
+        if params[:feature_group]
+          Feature.group(params[:feature_group])
+        elsif params[:user]
+          User.find_by_username(params[:user])
+        else
+          gate_value(params)
+        end
+      end
+    end
+
     resource :features do
       desc 'Get a list of all features' do
         success Entities::Feature
@@ -17,16 +40,22 @@ module API
       end
       params do
         requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+        optional :feature_group, type: String, desc: 'A Feature group name'
+        optional :user, type: String, desc: 'A GitLab username'
+        mutually_exclusive :feature_group, :user
       end
       post ':name' do
         feature = Feature.get(params[:name])
+        target = gate_target(params)
+        value = gate_value(params)
 
-        if %w(0 false).include?(params[:value])
-          feature.disable
-        elsif params[:value] == 'true'
-          feature.enable
+        case value
+        when true
+          feature.enable(target)
+        when false
+          feature.disable(target)
         else
-          feature.enable_percentage_of_time(params[:value].to_i)
+          feature.enable_percentage_of_time(value)
         end
 
         present feature, with: Entities::Feature, current_user: current_user
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 1369b021ea4..f8645e364ce 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -46,7 +46,8 @@ module API
 
         yield if block_given?
 
-        forbidden!('Project has been deleted!') unless job.project
+        project = job.project
+        forbidden!('Project has been deleted!') if project.nil? || project.pending_delete?
         forbidden!('Job has been erased!') if job.erased?
       end
 
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index 30761cb9b55..f1eaff6b0eb 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -17,7 +17,7 @@ module API
 
         namespaces = namespaces.search(params[:search]) if params[:search].present?
 
-        present paginate(namespaces), with: Entities::Namespace
+        present paginate(namespaces), with: Entities::Namespace, current_user: current_user
       end
     end
   end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index c5df45b7902..d0bd64b2972 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -1,3 +1,5 @@
+require_dependency 'declarative_policy'
+
 module API
   # Projects API
   class Projects < Grape::API
@@ -396,7 +398,7 @@ module API
         use :pagination
       end
       get ':id/users' do
-        users = user_project.team.users
+        users = DeclarativePolicy.subject_scope { user_project.team.users }
         users = users.search(params[:search]) if params[:search].present?
 
         present paginate(users), with: Entities::UserBasic
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 5109dc9670f..a40b6ab6c9f 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -28,7 +28,8 @@ module Ci
 
         yield if block_given?
 
-        forbidden!('Project has been deleted!') unless build.project
+        project = build.project
+        forbidden!('Project has been deleted!') if project.nil? || project.pending_delete?
         forbidden!('Build has been erased!') if build.erased?
       end
 
diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb
new file mode 100644
index 00000000000..d9959bc1aff
--- /dev/null
+++ b/lib/declarative_policy.rb
@@ -0,0 +1,58 @@
+require_dependency 'declarative_policy/cache'
+require_dependency 'declarative_policy/condition'
+require_dependency 'declarative_policy/dsl'
+require_dependency 'declarative_policy/preferred_scope'
+require_dependency 'declarative_policy/rule'
+require_dependency 'declarative_policy/runner'
+require_dependency 'declarative_policy/step'
+
+require_dependency 'declarative_policy/base'
+
+module DeclarativePolicy
+  class << self
+    def policy_for(user, subject, opts = {})
+      cache = opts[:cache] || {}
+      key = Cache.policy_key(user, subject)
+
+      cache[key] ||= class_for(subject).new(user, subject, opts)
+    end
+
+    def class_for(subject)
+      return GlobalPolicy if subject == :global
+      return NilPolicy if subject.nil?
+
+      subject = find_delegate(subject)
+
+      subject.class.ancestors.each do |klass|
+        next unless klass.name
+
+        begin
+          policy_class = "#{klass.name}Policy".constantize
+
+          # NOTE: the < operator here tests whether policy_class
+          # inherits from Base. We can't use #is_a? because that
+          # tests for *instances*, not *subclasses*.
+          return policy_class if policy_class < Base
+        rescue NameError
+          nil
+        end
+      end
+
+      raise "no policy for #{subject.class.name}"
+    end
+
+    private
+
+    def find_delegate(subject)
+      seen = Set.new
+
+      while subject.respond_to?(:declarative_policy_delegate)
+        raise ArgumentError, "circular delegations" if seen.include?(subject.object_id)
+        seen << subject.object_id
+        subject = subject.declarative_policy_delegate
+      end
+
+      subject
+    end
+  end
+end
diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb
new file mode 100644
index 00000000000..df94cafb6a1
--- /dev/null
+++ b/lib/declarative_policy/base.rb
@@ -0,0 +1,329 @@
+module DeclarativePolicy
+  class Base
+    # A map of ability => list of rules together with :enable
+    # or :prevent actions. Used to look up which rules apply to
+    # a given ability. See Base.ability_map
+    class AbilityMap
+      attr_reader :map
+      def initialize(map = {})
+        @map = map
+      end
+
+      # This merge behavior is different than regular hashes - if both
+      # share a key, the values at that key are concatenated, rather than
+      # overridden.
+      def merge(other)
+        conflict_proc = proc { |key, my_val, other_val| my_val + other_val }
+        AbilityMap.new(@map.merge(other.map, &conflict_proc))
+      end
+
+      def actions(key)
+        @map[key] ||= []
+      end
+
+      def enable(key, rule)
+        actions(key) << [:enable, rule]
+      end
+
+      def prevent(key, rule)
+        actions(key) << [:prevent, rule]
+      end
+    end
+
+    class << self
+      # The `own_ability_map` vs `ability_map` distinction is used so that
+      # the data structure is properly inherited - with subclasses recursively
+      # merging their parent class.
+      #
+      # This pattern is also used for conditions, global_actions, and delegations.
+      def ability_map
+        if self == Base
+          own_ability_map
+        else
+          superclass.ability_map.merge(own_ability_map)
+        end
+      end
+
+      def own_ability_map
+        @own_ability_map ||= AbilityMap.new
+      end
+
+      # an inheritable map of conditions, by name
+      def conditions
+        if self == Base
+          own_conditions
+        else
+          superclass.conditions.merge(own_conditions)
+        end
+      end
+
+      def own_conditions
+        @own_conditions ||= {}
+      end
+
+      # a list of global actions, generated by `prevent_all`. these aren't
+      # stored in `ability_map` because they aren't indexed by a particular
+      # ability.
+      def global_actions
+        if self == Base
+          own_global_actions
+        else
+          superclass.global_actions + own_global_actions
+        end
+      end
+
+      def own_global_actions
+        @own_global_actions ||= []
+      end
+
+      # an inheritable map of delegations, indexed by name (which may be
+      # autogenerated)
+      def delegations
+        if self == Base
+          own_delegations
+        else
+          superclass.delegations.merge(own_delegations)
+        end
+      end
+
+      def own_delegations
+        @own_delegations ||= {}
+      end
+
+      # all the [rule, action] pairs that apply to a particular ability.
+      # we combine the specific ones looked up in ability_map with the global
+      # ones.
+      def configuration_for(ability)
+        ability_map.actions(ability) + global_actions
+      end
+
+      ### declaration methods ###
+
+      def delegate(name = nil, &delegation_block)
+        if name.nil?
+          @delegate_name_counter ||= 0
+          @delegate_name_counter += 1
+          name = :"anonymous_#{@delegate_name_counter}"
+        end
+
+        name = name.to_sym
+
+        if delegation_block.nil?
+          delegation_block = proc { @subject.__send__(name) }
+        end
+
+        own_delegations[name] = delegation_block
+      end
+
+      # Declares a rule, constructed using RuleDsl, and returns
+      # a PolicyDsl which is used for registering the rule with
+      # this class. PolicyDsl will call back into Base.enable_when,
+      # Base.prevent_when, and Base.prevent_all_when.
+      def rule(&b)
+        rule = RuleDsl.new(self).instance_eval(&b)
+        PolicyDsl.new(self, rule)
+      end
+
+      # A hash in which to store calls to `desc` and `with_scope`, etc.
+      def last_options
+        @last_options ||= {}.with_indifferent_access
+      end
+
+      # retrieve and zero out the previously set options (used in .condition)
+      def last_options!
+        last_options.tap { @last_options = nil }
+      end
+
+      # Declare a description for the following condition. Currently unused,
+      # but opens the potential for explaining to users why they were or were
+      # not able to do something.
+      def desc(description)
+        last_options[:description] = description
+      end
+
+      def with_options(opts = {})
+        last_options.merge!(opts)
+      end
+
+      def with_scope(scope)
+        with_options scope: scope
+      end
+
+      def with_score(score)
+        with_options score: score
+      end
+
+      # Declares a condition. It gets stored in `own_conditions`, and generates
+      # a query method based on the condition's name.
+      def condition(name, opts = {}, &value)
+        name = name.to_sym
+
+        opts = last_options!.merge(opts)
+        opts[:context_key] ||= self.name
+
+        condition = Condition.new(name, opts, &value)
+
+        self.own_conditions[name] = condition
+
+        define_method(:"#{name}?") { condition(name).pass? }
+      end
+
+      # These next three methods are mainly called from PolicyDsl,
+      # and are responsible for "inverting" the relationship between
+      # an ability and a rule. We store in `ability_map` a map of
+      # abilities to rules that affect them, together with a
+      # symbol indicating :prevent or :enable.
+      def enable_when(abilities, rule)
+        abilities.each { |a| own_ability_map.enable(a, rule) }
+      end
+
+      def prevent_when(abilities, rule)
+        abilities.each { |a| own_ability_map.prevent(a, rule) }
+      end
+
+      # we store global prevents (from `prevent_all`) separately,
+      # so that they can be combined into every decision made.
+      def prevent_all_when(rule)
+        own_global_actions << [:prevent, rule]
+      end
+    end
+
+    # A policy object contains a specific user and subject on which
+    # to compute abilities. For this reason it's sometimes called
+    # "context" within the framework.
+    #
+    # It also stores a reference to the cache, so it can be used
+    # to cache computations by e.g. ManifestCondition.
+    attr_reader :user, :subject, :cache
+    def initialize(user, subject, opts = {})
+      @user = user
+      @subject = subject
+      @cache = opts[:cache] || {}
+    end
+
+    # helper for checking abilities on this and other subjects
+    # for the current user.
+    def can?(ability, new_subject = :_self)
+      return allowed?(ability) if new_subject == :_self
+
+      policy_for(new_subject).allowed?(ability)
+    end
+
+    # This is the main entry point for permission checks. It constructs
+    # or looks up a Runner for the given ability and asks it if it passes.
+    def allowed?(*abilities)
+      abilities.all? { |a| runner(a).pass? }
+    end
+
+    # The inverse of #allowed?, used mainly in specs.
+    def disallowed?(*abilities)
+      abilities.all? { |a| !runner(a).pass? }
+    end
+
+    # computes the given ability and prints a helpful debugging output
+    # showing which 
+    def debug(ability, *a)
+      runner(ability).debug(*a)
+    end
+
+    desc "Unknown user"
+    condition(:anonymous, scope: :user, score: 0) { @user.nil? }
+
+    desc "By default"
+    condition(:default, scope: :global, score: 0) { true }
+
+    def repr
+      subject_repr =
+        if @subject.respond_to?(:id)
+          "#{@subject.class.name}/#{@subject.id}"
+        else
+          @subject.inspect
+        end
+
+      user_repr =
+        if @user
+          @user.to_reference
+        else
+          "<anonymous>"
+        end
+
+      "(#{user_repr} : #{subject_repr})"
+    end
+
+    def inspect
+      "#<#{self.class.name} #{repr}>"
+    end
+
+    # returns a Runner for the given ability, capable of computing whether
+    # the ability is allowed. Runners are cached on the policy (which itself
+    # is cached on @cache), and caches its result. This is how we perform caching
+    # at the ability level.
+    def runner(ability)
+      ability = ability.to_sym
+      @runners ||= {}
+      @runners[ability] ||=
+        begin
+          delegated_runners = delegated_policies.values.compact.map { |p| p.runner(ability) }
+          own_runner = Runner.new(own_steps(ability))
+          delegated_runners.inject(own_runner, &:merge_runner)
+        end
+    end
+
+    # Helpers for caching. Used by ManifestCondition in performing condition
+    # computation.
+    #
+    # NOTE we can't use ||= here because the value might be the
+    # boolean `false`
+    def cache(key, &b)
+      return @cache[key] if cached?(key)
+      @cache[key] = yield
+    end
+
+    def cached?(key)
+      !@cache[key].nil?
+    end
+
+    # returns a ManifestCondition capable of computing itself. The computation
+    # will use our own @cache.
+    def condition(name)
+      name = name.to_sym
+      @_conditions ||= {}
+      @_conditions[name] ||=
+        begin
+          raise "invalid condition #{name}" unless self.class.conditions.key?(name)
+          ManifestCondition.new(self.class.conditions[name], self)
+        end
+    end
+
+    # used in specs - returns true if there is no possible way for any action
+    # to be allowed, determined only by the global :prevent_all rules.
+    def banned?
+      global_steps = self.class.global_actions.map { |(action, rule)| Step.new(self, rule, action) }
+      !Runner.new(global_steps).pass?
+    end
+
+    # A list of other policies that we've delegated to (see `Base.delegate`)
+    def delegated_policies
+      @delegated_policies ||= self.class.delegations.transform_values do |block|
+        new_subject = instance_eval(&block)
+
+        # never delegate to nil, as that would immediately prevent_all
+        next if new_subject.nil?
+
+        policy_for(new_subject)
+      end
+    end
+
+    def policy_for(other_subject)
+      DeclarativePolicy.policy_for(@user, other_subject, cache: @cache)
+    end
+
+    protected
+
+    # constructs steps that come from this policy and not from any delegations
+    def own_steps(ability)
+      rules = self.class.configuration_for(ability)
+      rules.map { |(action, rule)| Step.new(self, rule, action) }
+    end
+  end
+end
diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb
new file mode 100644
index 00000000000..b8cc60074c7
--- /dev/null
+++ b/lib/declarative_policy/cache.rb
@@ -0,0 +1,32 @@
+module DeclarativePolicy
+  module Cache
+    class << self
+      def user_key(user)
+        return '<anonymous>' if user.nil?
+        id_for(user)
+      end
+
+      def policy_key(user, subject)
+        u = user_key(user)
+        s = subject_key(subject)
+        "/dp/policy/#{u}/#{s}"
+      end
+
+      def subject_key(subject)
+        return '<nil>' if subject.nil?
+        return subject.inspect if subject.is_a?(Symbol)
+        "#{subject.class.name}:#{id_for(subject)}"
+      end
+
+      private
+
+      def id_for(obj)
+        if obj.respond_to?(:id) && obj.id
+          obj.id.to_s
+        else
+          "##{obj.object_id}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb
new file mode 100644
index 00000000000..9d7cf6b9726
--- /dev/null
+++ b/lib/declarative_policy/condition.rb
@@ -0,0 +1,102 @@
+module DeclarativePolicy
+  # A Condition is the data structure that is created by the
+  # `condition` declaration on DeclarativePolicy::Base. It is
+  # more or less just a struct of the data passed to that
+  # declaration. It holds on to the block to be instance_eval'd
+  # on a context (instance of Base) later, via #compute.
+  class Condition
+    attr_reader :name, :description, :scope
+    attr_reader :manual_score
+    attr_reader :context_key
+    def initialize(name, opts = {}, &compute)
+      @name = name
+      @compute = compute
+      @scope = opts.fetch(:scope, :normal)
+      @description = opts.delete(:description)
+      @context_key = opts[:context_key]
+      @manual_score = opts.fetch(:score, nil)
+    end
+
+    def compute(context)
+      !!context.instance_eval(&@compute)
+    end
+
+    def key
+      "#{@context_key}/#{@name}"
+    end
+  end
+
+  # In contrast to a Condition, a ManifestCondition contains
+  # a Condition and a context object, and is capable of calculating
+  # a result itself. This is the return value of Base#condition.
+  class ManifestCondition
+    def initialize(condition, context)
+      @condition = condition
+      @context = context
+    end
+
+    # The main entry point - does this condition pass? We reach into
+    # the context's cache here so that we can share in the global
+    # cache (often RequestStore or similar).
+    def pass?
+      @context.cache(cache_key) { @condition.compute(@context) }
+    end
+
+    # Whether we've already computed this condition.
+    def cached?
+      @context.cached?(cache_key)
+    end
+
+    # This is used to score Rule::Condition. See Rule::Condition#score
+    # and Runner#steps_by_score for how scores are used.
+    #
+    # The number here is intended to represent, abstractly, how
+    # expensive it would be to calculate this condition.
+    #
+    # See #cache_key for info about @condition.scope.
+    def score
+      # If we've been cached, no computation is necessary.
+      return 0 if cached?
+
+      # Use the override from condition(score: ...) if present
+      return @condition.manual_score if @condition.manual_score
+
+      # Global scope rules are cheap due to max cache sharing
+      return 2 if  @condition.scope == :global
+
+      # "Normal" rules can't share caches with any other policies
+      return 16 if @condition.scope == :normal
+
+      # otherwise, we're :user or :subject scope, so it's 4 if
+      # the caller has declared a preference
+      return 4 if @condition.scope == DeclarativePolicy.preferred_scope
+
+      # and 8 for all other :user or :subject scope conditions.
+      8
+    end
+
+    private
+
+    # This method controls the caching for the condition. This is where
+    # the condition(scope: ...) option comes into play. Notice that
+    # depending on the scope, we may cache only by the user or only by
+    # the subject, resulting in sharing across different policy objects.
+    def cache_key
+      case @condition.scope
+      when :normal  then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}"
+      when :user    then "/dp/condition/#{@condition.key}/#{user_key}"
+      when :subject then "/dp/condition/#{@condition.key}/#{subject_key}"
+      when :global  then "/dp/condition/#{@condition.key}"
+      else raise 'invalid scope'
+      end
+    end
+
+    def user_key
+      Cache.user_key(@context.user)
+    end
+
+    def subject_key
+      Cache.subject_key(@context.subject)
+    end
+  end
+end
diff --git a/lib/declarative_policy/dsl.rb b/lib/declarative_policy/dsl.rb
new file mode 100644
index 00000000000..b26807a7622
--- /dev/null
+++ b/lib/declarative_policy/dsl.rb
@@ -0,0 +1,103 @@
+module DeclarativePolicy
+  # The DSL evaluation context inside rule { ... } blocks.
+  # Responsible for creating and combining Rule objects.
+  #
+  # See Base.rule
+  class RuleDsl
+    def initialize(context_class)
+      @context_class = context_class
+    end
+
+    def can?(ability)
+      Rule::Ability.new(ability)
+    end
+
+    def all?(*rules)
+      Rule::And.make(rules)
+    end
+
+    def any?(*rules)
+      Rule::Or.make(rules)
+    end
+
+    def none?(*rules)
+      ~Rule::Or.new(rules)
+    end
+
+    def cond(condition)
+      Rule::Condition.new(condition)
+    end
+
+    def delegate(delegate_name, condition)
+      Rule::DelegatedCondition.new(delegate_name, condition)
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless a.size == 0 && !block_given?
+
+      if @context_class.delegations.key?(m)
+        DelegateDsl.new(self, m)
+      else
+        cond(m.to_sym)
+      end
+    end
+  end
+
+  # Used when the name of a delegate is mentioned in
+  # the rule DSL.
+  class DelegateDsl
+    def initialize(rule_dsl, delegate_name)
+      @rule_dsl = rule_dsl
+      @delegate_name = delegate_name
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless a.size == 0 && !block_given?
+
+      @rule_dsl.delegate(@delegate_name, m)
+    end
+  end
+
+  # The return value of a rule { ... } declaration.
+  # Can call back to register rules with the containing
+  # Policy class (context_class here). See Base.rule
+  #
+  # Note that the #policy method just performs an #instance_eval,
+  # which is useful for multiple #enable or #prevent callse.
+  #
+  # Also provides a #method_missing proxy to the context
+  # class's class methods, so that helper methods can be
+  # defined and used in a #policy { ... } block.
+  class PolicyDsl
+    def initialize(context_class, rule)
+      @context_class = context_class
+      @rule = rule
+    end
+
+    def policy(&b)
+      instance_eval(&b)
+    end
+
+    def enable(*abilities)
+      @context_class.enable_when(abilities, @rule)
+    end
+
+    def prevent(*abilities)
+      @context_class.prevent_when(abilities, @rule)
+    end
+
+    def prevent_all
+      @context_class.prevent_all_when(@rule)
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless @context_class.respond_to?(m)
+
+      @context_class.__send__(m, *a, &b)
+    end
+
+    def respond_to_missing?(m)
+      @context_class.respond_to?(m) || super
+    end
+  end
+end
diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb
new file mode 100644
index 00000000000..b0754098149
--- /dev/null
+++ b/lib/declarative_policy/preferred_scope.rb
@@ -0,0 +1,28 @@
+module DeclarativePolicy
+  PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope"
+
+  class << self
+    def with_preferred_scope(scope, &b)
+      Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY]
+      yield
+    ensure
+      Thread.current[PREFERRED_SCOPE_KEY] = old_scope
+    end
+
+    def preferred_scope
+      Thread.current[PREFERRED_SCOPE_KEY]
+    end
+
+    def user_scope(&b)
+      with_preferred_scope(:user, &b)
+    end
+
+    def subject_scope(&b)
+      with_preferred_scope(:subject, &b)
+    end
+
+    def preferred_scope=(scope)
+      Thread.current[PREFERRED_SCOPE_KEY] = scope
+    end
+  end
+end
diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb
new file mode 100644
index 00000000000..bfcec241489
--- /dev/null
+++ b/lib/declarative_policy/rule.rb
@@ -0,0 +1,301 @@
+module DeclarativePolicy
+  module Rule
+    # A Rule is the object that results from the `rule` declaration,
+    # usually built using the DSL in `RuleDsl`. It is a basic logical
+    # combination of building blocks, and is capable of deciding,
+    # given a context (instance of DeclarativePolicy::Base) whether it
+    # passes or not. Note that this decision doesn't by itself know
+    # how that affects the actual ability decision - for that, a
+    # `Step` is used.
+    class Base
+      def self.make(*a)
+        new(*a).simplify
+      end
+
+      # true or false whether this rule passes.
+      # `context` is a policy - an instance of
+      # DeclarativePolicy::Base.
+      def pass?(context)
+        raise 'abstract'
+      end
+
+      # same as #pass? except refuses to do any I/O,
+      # returning nil if the result is not yet cached.
+      # used for accurately scoring And/Or
+      def cached_pass?(context)
+        raise 'abstract'
+      end
+
+      # abstractly, how long would it take to compute
+      # this rule? lower-scored rules are tried first.
+      def score(context)
+        raise 'abstract'
+      end
+
+      # unwrap double negatives and nested and/or
+      def simplify
+        self
+      end
+
+      # convenience combination methods
+      def or(other)
+        Or.make([self, other])
+      end
+
+      def and(other)
+        And.make([self, other])
+      end
+
+      def negate
+        Not.make(self)
+      end
+
+      alias_method :|, :or
+      alias_method :&, :and
+      alias_method :~@, :negate
+
+      def inspect
+        "#<Rule #{repr}>"
+      end
+    end
+
+    # A rule that checks a condition. This is the
+    # type of rule that results from a basic bareword
+    # in the rule dsl (see RuleDsl#method_missing).
+    class Condition < Base
+      def initialize(name)
+        @name = name
+      end
+
+      # we delegate scoring to the condition. See
+      # ManifestCondition#score.
+      def score(context)
+        context.condition(@name).score
+      end
+
+      # Let the ManifestCondition from the context
+      # decide whether we pass.
+      def pass?(context)
+        context.condition(@name).pass?
+      end
+
+      # returns nil unless it's already cached
+      def cached_pass?(context)
+        condition = context.condition(@name)
+        return nil unless condition.cached?
+        condition.pass?
+      end
+
+      def description(context)
+        context.class.conditions[@name].description
+      end
+
+      def repr
+        @name.to_s
+      end
+    end
+
+    # A rule constructed from DelegateDsl - using a condition from a
+    # delegated policy.
+    class DelegatedCondition < Base
+      # Internal use only - this is rescued each time it's raised.
+      MissingDelegate = Class.new(StandardError)
+
+      def initialize(delegate_name, name)
+        @delegate_name = delegate_name
+        @name = name
+      end
+
+      def delegated_context(context)
+        policy = context.delegated_policies[@delegate_name]
+        raise MissingDelegate if policy.nil?
+        policy
+      end
+
+      def score(context)
+        delegated_context(context).condition(@name).score
+      rescue MissingDelegate
+        0
+      end
+
+      def cached_pass?(context)
+        condition = delegated_context(context).condition(@name)
+        return nil unless condition.cached?
+        condition.pass?
+      rescue MissingDelegate
+        false
+      end
+
+      def pass?(context)
+        delegated_context(context).condition(@name).pass?
+      rescue MissingDelegate
+        false
+      end
+
+      def repr
+        "#{@delegate_name}.#{@name}"
+      end
+    end
+
+    # A rule constructed from RuleDsl#can?. Computes a different ability
+    # on the same subject.
+    class Ability < Base
+      attr_reader :ability
+      def initialize(ability)
+        @ability = ability
+      end
+
+      # We ask the ability's runner for a score
+      def score(context)
+        context.runner(@ability).score
+      end
+
+      def pass?(context)
+        context.allowed?(@ability)
+      end
+
+      def cached_pass?(context)
+        runner = context.runner(@ability)
+        return nil unless runner.cached?
+        runner.pass?
+      end
+
+      def description(context)
+        "User can #{@ability.inspect}"
+      end
+
+      def repr
+        "can?(#{@ability.inspect})"
+      end
+    end
+
+    # Logical `and`, containing a list of rules. Only passes
+    # if all of them do.
+    class And < Base
+      attr_reader :rules
+      def initialize(rules)
+        @rules = rules
+      end
+
+      def simplify
+        simplified_rules = @rules.flat_map do |rule|
+          simplified = rule.simplify
+          case simplified
+          when And then simplified.rules
+          else [simplified]
+          end
+        end
+
+        And.new(simplified_rules)
+      end
+
+      def score(context)
+        return 0 unless cached_pass?(context).nil?
+
+        # note that cached rules will have score 0 anyways.
+        @rules.map { |r| r.score(context) }.inject(0, :+)
+      end
+
+      def pass?(context)
+        # try to find a cached answer before
+        # checking in order
+        cached = cached_pass?(context)
+        return cached unless cached.nil?
+
+        @rules.all? { |r| r.pass?(context) }
+      end
+
+      def cached_pass?(context)
+        passes = @rules.map { |r| r.cached_pass?(context) }
+        return false if passes.any? { |p| p == false }
+        return true if passes.all? { |p| p == true }
+
+        nil
+      end
+
+      def repr
+        "all?(#{rules.map(&:repr).join(', ')})"
+      end
+    end
+
+    # Logical `or`. Mirrors And.
+    class Or < Base
+      attr_reader :rules
+      def initialize(rules)
+        @rules = rules
+      end
+
+      def pass?(context)
+        cached = cached_pass?(context)
+        return cached unless cached.nil?
+
+        @rules.any? { |r| r.pass?(context) }
+      end
+
+      def simplify
+        simplified_rules = @rules.flat_map do |rule|
+          simplified = rule.simplify
+          case simplified
+          when Or then simplified.rules
+          else [simplified]
+          end
+        end
+
+        Or.new(simplified_rules)
+      end
+
+      def cached_pass?(context)
+        passes = @rules.map { |r| r.cached_pass?(context) }
+        return true if passes.any? { |p| p == true }
+        return false if passes.all? { |p| p == false }
+
+        nil
+      end
+
+      def score(context)
+        return 0 unless cached_pass?(context).nil?
+        @rules.map { |r| r.score(context) }.inject(0, :+)
+      end
+
+      def repr
+        "any?(#{@rules.map(&:repr).join(', ')})"
+      end
+    end
+
+    class Not < Base
+      attr_reader :rule
+      def initialize(rule)
+        @rule = rule
+      end
+
+      def simplify
+        case @rule
+        when And then Or.new(@rule.rules.map(&:negate)).simplify
+        when Or then And.new(@rule.rules.map(&:negate)).simplify
+        when Not then @rule.rule.simplify
+        else Not.new(@rule.simplify)
+        end
+      end
+
+      def pass?(context)
+        !@rule.pass?(context)
+      end
+
+      def cached_pass?(context)
+        case @rule.cached_pass?(context)
+        when nil then nil
+        when true then false
+        when false then true
+        end
+      end
+
+      def score(context)
+        @rule.score(context)
+      end
+
+      def repr
+        "~#{@rule.repr}"
+      end
+    end
+  end
+end
diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb
new file mode 100644
index 00000000000..b5c615da4e3
--- /dev/null
+++ b/lib/declarative_policy/runner.rb
@@ -0,0 +1,181 @@
+module DeclarativePolicy
+  class Runner
+    class State
+      def initialize
+        @enabled = false
+        @prevented = false
+      end
+
+      def enable!
+        @enabled = true
+      end
+
+      def enabled?
+        @enabled
+      end
+
+      def prevent!
+        @prevented = true
+      end
+
+      def prevented?
+        @prevented
+      end
+
+      def pass?
+        !prevented? && enabled?
+      end
+    end
+
+    # a Runner contains a list of Steps to be run.
+    attr_reader :steps
+    def initialize(steps)
+      @steps = steps
+    end
+
+    # We make sure only to run any given Runner once,
+    # and just continue to use the resulting @state
+    # that's left behind.
+    def cached?
+      !!@state
+    end
+
+    # used by Rule::Ability. See #steps_by_score
+    def score
+      return 0 if cached?
+      steps.map(&:score).inject(0, :+)
+    end
+
+    def merge_runner(other)
+      Runner.new(@steps + other.steps)
+    end
+
+    # The main entry point, called for making an ability decision.
+    # See #run and DeclarativePolicy::Base#can?
+    def pass?
+      run unless cached?
+
+      @state.pass?
+    end
+
+    # see DeclarativePolicy::Base#debug
+    def debug(out = $stderr)
+      run(out)
+    end
+
+    private
+
+    def flatten_steps!
+      @steps = @steps.flat_map { |s| s.flattened(@steps) }
+    end
+
+    # This method implements the semantic of "one enable and no prevents".
+    # It relies on #steps_by_score for the main loop, and updates @state
+    # with the result of the step.
+    def run(debug = nil)
+      @state = State.new
+
+      steps_by_score do |step, score|
+        passed = nil
+        case step.action
+        when :enable then
+          # we only check :enable actions if they have a chance of
+          # changing the outcome - if no other rule has enabled or
+          # prevented.
+          unless @state.enabled? || @state.prevented?
+            passed = step.pass?
+            @state.enable! if passed
+          end
+
+          debug << inspect_step(step, score, passed) if debug
+        when :prevent then
+          # we only check :prevent actions if the state hasn't already
+          # been prevented.
+          unless @state.prevented?
+            passed = step.pass?
+            if passed
+              @state.prevent!
+              return unless debug
+            end
+          end
+
+          debug << inspect_step(step, score, passed) if debug
+        else raise "invalid action #{step.action.inspect}"
+        end
+      end
+
+      @state
+    end
+
+    # This is the core spot where all those `#score` methods matter.
+    # It is critcal for performance to run steps in the correct order,
+    # so that we don't compute expensive conditions (potentially n times
+    # if we're called on, say, a large list of users).
+    #
+    # In order to determine the cheapest step to run next, we rely on
+    # Step#score, which returns a numerical rating of how expensive
+    # it would be to calculate - the lower the better. It would be
+    # easy enough to statically sort by these scores, but we can do
+    # a little better - the scores are cache-aware (conditions that
+    # are already in the cache have score 0), which means that running
+    # a step can actually change the scores of other steps.
+    #
+    # So! The way we sort here involves re-scoring at every step. This
+    # is by necessity quadratic, but most of the time the number of steps
+    # will be low. But just in case, if the number of steps exceeds 50,
+    # we print a warning and fall back to a static sort.
+    #
+    # For each step, we yield the step object along with the computed score
+    # for debugging purposes.
+    def steps_by_score(&b)
+      flatten_steps!
+
+      if @steps.size > 50
+        warn "DeclarativePolicy: large number of steps (#{steps.size}), falling back to static sort"
+
+        @steps.map { |s| [s.score, s] }.sort_by { |(score, _)| score }.each do |(score, step)|
+          yield step, score
+        end
+
+        return
+      end
+
+      steps = Set.new(@steps)
+
+      loop do
+        return if steps.empty?
+
+        # if the permission hasn't yet been enabled and we only have
+        # prevent steps left, we short-circuit the state here
+        @state.prevent! if !@state.enabled? && steps.all?(&:prevent?)
+
+        lowest_score = Float::INFINITY
+        next_step = nil
+
+        steps.each do |step|
+          score = step.score
+          if score < lowest_score
+            next_step = step
+            lowest_score = score
+          end
+        end
+
+        steps.delete(next_step)
+
+        yield next_step, lowest_score
+      end
+    end
+
+    # Formatter for debugging output.
+    def inspect_step(step, original_score, passed)
+      symbol =
+        case passed
+        when true then '+'
+        when false then '-'
+        when nil then ' '
+        end
+
+      "#{symbol} [#{original_score.to_i}] #{step.repr}\n"
+    end
+  end
+end
diff --git a/lib/declarative_policy/step.rb b/lib/declarative_policy/step.rb
new file mode 100644
index 00000000000..3469fe9f991
--- /dev/null
+++ b/lib/declarative_policy/step.rb
@@ -0,0 +1,86 @@
+module DeclarativePolicy
+  # This object represents one step in the runtime decision of whether
+  # an ability is allowed. It contains a Rule and a context (instance
+  # of DeclarativePolicy::Base), which contains the user, the subject,
+  # and the cache. It also contains an "action", which is the symbol
+  # :prevent or :enable.
+  class Step
+    attr_reader :context, :rule, :action
+    def initialize(context, rule, action)
+      @context = context
+      @rule = rule
+      @action = action
+    end
+
+    # In the flattening process, duplicate steps may be generated in the
+    # same rule. This allows us to eliminate those (see Runner#steps_by_score
+    # and note its use of a Set)
+    def ==(other)
+      @context == other.context && @rule == other.rule && @action == other.action
+    end
+
+    # In the runner, steps are sorted dynamically by score, so that
+    # we are sure to compute them in close to the optimal order.
+    #
+    # See also Rule#score, ManifestCondition#score, and Runner#steps_by_score.
+    def score
+      # we slightly prefer the preventative actions
+      # since they are more likely to short-circuit
+      case @action
+      when :prevent
+        @rule.score(@context) * (7.0 / 8)
+      when :enable
+        @rule.score(@context)
+      end
+    end
+
+    def with_action(action)
+      Step.new(@context, @rule, action)
+    end
+
+    def enable?
+      @action == :enable
+    end
+
+    def prevent?
+      @action == :prevent
+    end
+
+    # This rather complex method allows us to split rules into parts so that
+    # they can be sorted independently for better optimization
+    def flattened(roots)
+      case @rule
+      when Rule::Or
+        # A single `Or` step is the same as each of its elements as separate steps
+        @rule.rules.flat_map { |r| Step.new(@context, r, @action).flattened(roots) }
+      when Rule::Ability
+        # This looks like a weird micro-optimization but it buys us quite a lot
+        # in some cases. If we depend on an Ability (i.e. a `can?(...)` rule),
+        # and that ability *only* has :enable actions (modulo some actions that
+        # we already have taken care of), then its rules can be safely inlined.
+        steps = @context.runner(@rule.ability).steps.reject { |s| roots.include?(s) }
+
+        if steps.all?(&:enable?)
+          # in the case that we are a :prevent step, each inlined step becomes
+          # an independent :prevent, even though it was an :enable in its initial
+          # context.
+          steps.map! { |s| s.with_action(:prevent) } if prevent?
+
+          steps.flat_map { |s| s.flattened(roots) }
+        else
+          [self]
+        end
+      else
+        [self]
+      end
+    end
+
+    def pass?
+      @rule.pass?(@context)
+    end
+
+    def repr
+      "#{@action} when #{@rule.repr} (#{@context.repr})"
+    end
+  end
+end
diff --git a/lib/feature.rb b/lib/feature.rb
index d3d972564af..363f66ba60e 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -12,6 +12,8 @@ class Feature
   end
 
   class << self
+    delegate :group, to: :flipper
+
     def all
       flipper.features.to_a
     end
@@ -27,16 +29,24 @@ class Feature
       all.map(&:name).include?(feature.name)
     end
 
-    def enabled?(key)
-      get(key).enabled?
+    def enabled?(key, thing = nil)
+      get(key).enabled?(thing)
+    end
+
+    def enable(key, thing = true)
+      get(key).enable(thing)
+    end
+
+    def disable(key, thing = false)
+      get(key).disable(thing)
     end
 
-    def enable(key)
-      get(key).enable
+    def enable_group(key, group)
+      get(key).enable_group(group)
     end
 
-    def disable(key)
-      get(key).disable
+    def disable_group(key, group)
+      get(key).disable_group(group)
     end
 
     def flipper
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb
index e4f7cad2b79..45c2b01dd8f 100644
--- a/lib/gitlab/allowable.rb
+++ b/lib/gitlab/allowable.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Allowable
-    def can?(user, action, subject = :global)
-      Ability.allowed?(user, action, subject)
+    def can?(*args)
+      Ability.allowed?(*args)
     end
   end
 end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
index 89530082cd2..f333ff22300 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
@@ -29,6 +29,11 @@ module Gitlab
           paths = Array(paths)
           RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
         end
+
+        def revert_renames
+          RenameProjects.new([], self).revert_renames
+          RenameNamespaces.new([], self).revert_renames
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index d8163d7da11..33f8939bc61 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -6,7 +6,10 @@ module Gitlab
           attr_reader :paths, :migration
 
           delegate :update_column_in_batches,
+                   :execute,
                    :replace_sql,
+                   :quote_string,
+                   :say,
                    to: :migration
 
           def initialize(paths, migration)
@@ -26,24 +29,45 @@ module Gitlab
             new_path = rename_path(namespace_path, old_path)
             new_full_path = join_routable_path(namespace_path, new_path)
 
+            perform_rename(routable, old_full_path, new_full_path)
+
+            [old_full_path, new_full_path]
+          end
+
+          def perform_rename(routable, old_full_path, new_full_path)
             # skips callbacks & validations
+            new_path = new_full_path.split('/').last
             routable.class.where(id: routable)
               .update_all(path: new_path)
 
             rename_routes(old_full_path, new_full_path)
-
-            [old_full_path, new_full_path]
           end
 
           def rename_routes(old_full_path, new_full_path)
+            routes = Route.arel_table
+
+            quoted_old_full_path = quote_string(old_full_path)
+            quoted_old_wildcard_path = quote_string("#{old_full_path}/%")
+
+            filter = if Database.mysql?
+                       "lower(routes.path) = lower('#{quoted_old_full_path}') "\
+                       "OR routes.path LIKE '#{quoted_old_wildcard_path}'"
+                     else
+                       "routes.id IN "\
+                       "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\
+                       "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )"
+                     end
+
             replace_statement = replace_sql(Route.arel_table[:path],
                                             old_full_path,
                                             new_full_path)
 
-            update_column_in_batches(:routes, :path, replace_statement)  do |table, query|
-              path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"])
-              query.where(path_or_children)
-            end
+            update = Arel::UpdateManager.new(ActiveRecord::Base)
+                       .table(routes)
+                       .set([[routes[:path], replace_statement]])
+                       .where(Arel::Nodes::SqlLiteral.new(filter))
+
+            execute(update.to_sql)
           end
 
           def rename_path(namespace_path, path_was)
@@ -86,32 +110,74 @@ module Gitlab
 
           def move_folders(directory, old_relative_path, new_relative_path)
             old_path = File.join(directory, old_relative_path)
-            return unless File.directory?(old_path)
+            unless File.directory?(old_path)
+              say "#{old_path} doesn't exist, skipping"
+              return
+            end
 
             new_path = File.join(directory, new_relative_path)
             FileUtils.mv(old_path, new_path)
           end
 
           def remove_cached_html_for_projects(project_ids)
-            update_column_in_batches(:projects, :description_html, nil) do |table, query|
-              query.where(table[:id].in(project_ids))
-            end
-
-            update_column_in_batches(:issues, :description_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
+            project_ids.each do |project_id|
+              update_column_in_batches(:projects, :description_html, nil) do |table, query|
+                query.where(table[:id].eq(project_id))
+              end
+
+              update_column_in_batches(:issues, :description_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
+                query.where(table[:target_project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:notes, :note_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:milestones, :description_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
             end
+          end
 
-            update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
-              query.where(table[:target_project_id].in(project_ids))
+          def track_rename(type, old_path, new_path)
+            key = redis_key_for_type(type)
+            Gitlab::Redis.with do |redis|
+              redis.lpush(key, [old_path, new_path].to_json)
+              redis.expire(key, 2.weeks.to_i)
             end
+            say "tracked rename: #{key}: #{old_path} -> #{new_path}"
+          end
 
-            update_column_in_batches(:notes, :note_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
+          def reverts_for_type(type)
+            key = redis_key_for_type(type)
+
+            Gitlab::Redis.with do |redis|
+              failed_reverts = []
+
+              while rename_info = redis.lpop(key)
+                path_before_rename, path_after_rename = JSON.parse(rename_info)
+                say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}"
+                begin
+                  yield(path_before_rename, path_after_rename)
+                rescue StandardError => e
+                  failed_reverts << rename_info
+                  say "Renaming #{type} from #{path_after_rename} back to "\
+                      "#{path_before_rename} failed. Review the error and try "\
+                      "again by running the `down` action. \n"\
+                      "#{e.message}: \n #{e.backtrace.join("\n")}"
+                end
+              end
+
+              failed_reverts.each { |rename_info| redis.lpush(key, rename_info) }
             end
+          end
 
-            update_column_in_batches(:milestones, :description_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
-            end
+          def redis_key_for_type(type)
+            "rename:#{migration.name}:#{type}"
           end
 
           def file_storage?
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
index da7e2cb2e85..05b86f32ce2 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -26,6 +26,12 @@ module Gitlab
           def rename_namespace(namespace)
             old_full_path, new_full_path = rename_path_for_routable(namespace)
 
+            track_rename('namespace', old_full_path, new_full_path)
+
+            rename_namespace_dependencies(namespace, old_full_path, new_full_path)
+          end
+
+          def rename_namespace_dependencies(namespace, old_full_path, new_full_path)
             move_repositories(namespace, old_full_path, new_full_path)
             move_uploads(old_full_path, new_full_path)
             move_pages(old_full_path, new_full_path)
@@ -33,6 +39,23 @@ module Gitlab
             remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
           end
 
+          def revert_renames
+            reverts_for_type('namespace') do |path_before_rename, current_path|
+              matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
+              namespace = MigrationClasses::Namespace.joins(:route)
+                            .where(matches_path).first&.becomes(MigrationClasses::Namespace)
+
+              if namespace
+                perform_rename(namespace, current_path, path_before_rename)
+
+                rename_namespace_dependencies(namespace, current_path, path_before_rename)
+              else
+                say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\
+                    "namespace was renamed, or no longer exists at the expected path"
+              end
+            end
+          end
+
           def rename_user(old_username, new_username)
             MigrationClasses::User.where(username: old_username)
               .update_all(username: new_username)
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
index 448717eb744..75a75f61953 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
@@ -16,12 +16,37 @@ module Gitlab
           def rename_project(project)
             old_full_path, new_full_path = rename_path_for_routable(project)
 
+            track_rename('project', old_full_path, new_full_path)
+
+            move_project_folders(project, old_full_path, new_full_path)
+          end
+
+          def move_project_folders(project, old_full_path, new_full_path)
             move_repository(project, old_full_path, new_full_path)
             move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
             move_uploads(old_full_path, new_full_path)
             move_pages(old_full_path, new_full_path)
           end
 
+          def revert_renames
+            reverts_for_type('project') do |path_before_rename, current_path|
+              matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
+              project = MigrationClasses::Project.joins(:route)
+                          .where(matches_path).first
+
+              if project
+                perform_rename(project, current_path, path_before_rename)
+
+                move_project_folders(project, current_path, path_before_rename)
+              else
+                say "Couldn't rename project from #{current_path} back to "\
+                    "#{path_before_rename}, project was renamed or no longer "\
+                    "exists at the expected path."
+
+              end
+            end
+          end
+
           def move_repository(project, old_path, new_path)
             unless gitlab_shell.mv_repository(project.repository_storage_path,
                                               old_path,
diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb
new file mode 100644
index 00000000000..d9400e04b83
--- /dev/null
+++ b/lib/gitlab/database/sha_attribute.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Database
+    BINARY_TYPE = if Gitlab::Database.postgresql?
+                    # PostgreSQL defines its own class with slightly different
+                    # behaviour from the default Binary type.
+                    ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
+                  else
+                    ActiveRecord::Type::Binary
+                  end
+
+    # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa).
+    #
+    # Using ShaAttribute allows you to store SHA1 values as binary while still
+    # using them as if they were stored as string values. This gives you the
+    # ease of use of string values, but without the storage overhead.
+    class ShaAttribute < BINARY_TYPE
+      PACK_FORMAT = 'H*'.freeze
+
+      # Casts binary data to a SHA1 in hexadecimal.
+      def type_cast_from_database(value)
+        value = super
+
+        value ? value.unpack(PACK_FORMAT)[0] : nil
+      end
+
+      # Casts a SHA1 in hexadecimal to the proper binary format.
+      def type_cast_for_database(value)
+        arg = value ? [value].pack(PACK_FORMAT) : nil
+
+        super(arg)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb
index 7bbd154eb03..d2360583741 100644
--- a/lib/gitlab/dependency_linker/base_linker.rb
+++ b/lib/gitlab/dependency_linker/base_linker.rb
@@ -52,7 +52,7 @@ module Gitlab
       #   # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"`
       def link_regex(regex, &url_proc)
         highlighted_lines.map!.with_index do |rich_line, i|
-          marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe)
+          marker = StringRegexMarker.new(plain_lines[i].chomp, rich_line.html_safe)
 
           marker.mark(regex, group: :name) do |text, left:, right:|
             url = yield(text)
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 936606152e9..4175746be39 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -7,8 +7,10 @@ module Gitlab
     CommandError = Class.new(StandardError)
 
     class << self
+      include Gitlab::EncodingHelper
+
       def ref_name(ref)
-        ref.sub(/\Arefs\/(tags|heads)\//, '')
+        encode! ref.sub(/\Arefs\/(tags|heads)\//, '')
       end
 
       def branch_name(ref)
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index bd90d24a2ec..5042916343b 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -4,9 +4,10 @@ module Gitlab
       GL_PROTOCOL = 'web'.freeze
       attr_reader :name, :repo_path, :path
 
-      def initialize(name, repo_path)
+      def initialize(name, project)
         @name = name
-        @repo_path = repo_path
+        @project = project
+        @repo_path = project.repository.path
         @path = File.join(repo_path.strip, 'hooks', name)
       end
 
@@ -38,7 +39,8 @@ module Gitlab
         vars = {
           'GL_ID' => gl_id,
           'PWD' => repo_path,
-          'GL_PROTOCOL' => GL_PROTOCOL
+          'GL_PROTOCOL' => GL_PROTOCOL,
+          'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false)
         }
 
         options = {
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 0a0c6f76cd3..23d0c8a9bdb 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -113,9 +113,7 @@ module Gitlab
       def local_branches(sort_by: nil)
         gitaly_migrate(:local_branches) do |is_enabled|
           if is_enabled
-            gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch|
-              Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch)
-            end
+            gitaly_ref_client.local_branches(sort_by: sort_by)
           else
             branches(filter: :local, sort_by: sort_by)
           end
@@ -549,32 +547,20 @@ module Gitlab
         rugged.rev_parse(oid_or_ref_name)
       end
 
-      # Return hash with submodules info for this repository
+      # Returns url for submodule
       #
       # Ex.
-      #   {
-      #     "current_path/rack"  => {
-      #       "name" => "original_path/rack",
-      #       "id" => "c67be4624545b4263184c4a0e8f887efd0a66320",
-      #       "url" => "git://github.com/chneukirchen/rack.git"
-      #     },
-      #     "encoding" => {
-      #       "id" => ....
-      #     }
-      #   }
+      #   @repository.submodule_url_for('master', 'rack')
+      #   # => git@localhost:rack.git
       #
-      def submodules(ref)
-        commit = rev_parse_target(ref)
-        return {} unless commit
+      def submodule_url_for(ref, path)
+        if submodules(ref).any?
+          submodule = submodules(ref)[path]
 
-        begin
-          content = blob_content(commit, ".gitmodules")
-        rescue InvalidBlobName
-          return {}
+          if submodule
+            submodule['url']
+          end
         end
-
-        parser = GitmodulesParser.new(content)
-        fill_submodule_ids(commit, parser.parse)
       end
 
       # Return total commits count accessible from passed ref
@@ -912,6 +898,23 @@ module Gitlab
 
       private
 
+      # We are trying to deprecate this method because it does a lot of work
+      # but it seems to be used only to look up submodule URL's.
+      # https://gitlab.com/gitlab-org/gitaly/issues/329
+      def submodules(ref)
+        commit = rev_parse_target(ref)
+        return {} unless commit
+
+        begin
+          content = blob_content(commit, ".gitmodules")
+        rescue InvalidBlobName
+          return {}
+        end
+
+        parser = GitmodulesParser.new(content)
+        fill_submodule_ids(commit, parser.parse)
+      end
+
       def alternate_object_directories
         Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact
       end
diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb
index 6d5f54dd959..2d61992f595 100644
--- a/lib/gitlab/gitaly_client/ref.rb
+++ b/lib/gitlab/gitaly_client/ref.rb
@@ -1,8 +1,11 @@
 module Gitlab
   module GitalyClient
     class Ref
+      include Gitlab::EncodingHelper
+
       # 'repository' is a Gitlab::Git::Repository
       def initialize(repository)
+        @repository = repository
         @gitaly_repo = repository.gitaly_repository
         @storage = repository.storage
       end
@@ -16,13 +19,13 @@ module Gitlab
       def branch_names
         request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo)
         response = GitalyClient.call(@storage, :ref, :find_all_branch_names, request)
-        consume_refs_response(response, prefix: 'refs/heads/')
+        consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) }
       end
 
       def tag_names
         request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo)
         response = GitalyClient.call(@storage, :ref, :find_all_tag_names, request)
-        consume_refs_response(response, prefix: 'refs/tags/')
+        consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) }
       end
 
       def find_ref_name(commit_id, ref_prefix)
@@ -51,20 +54,28 @@ module Gitlab
 
       private
 
-      def consume_refs_response(response, prefix:)
-        response.flat_map do |r|
-          r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') }
-        end
+      def consume_refs_response(response)
+        response.flat_map { |message| message.names.map { |name| yield(name) } }
       end
 
       def sort_by_param(sort_by)
+        sort_by = 'name' if sort_by == 'name_asc'
+
         enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym)
         raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value
         enum_value
       end
 
       def consume_branches_response(response)
-        response.flat_map { |r| r.branches }
+        response.flat_map do |message|
+          message.branches.map do |gitaly_branch|
+            Gitlab::Git::Branch.new(
+              @repository,
+              encode!(gitaly_branch.name.dup),
+              gitaly_branch.commit_id
+            )
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb
index 8779577258b..fb68627dedf 100644
--- a/lib/gitlab/ldap/access.rb
+++ b/lib/gitlab/ldap/access.rb
@@ -16,7 +16,7 @@ module Gitlab
       def self.allowed?(user)
         self.open(user) do |access|
           if access.allowed?
-            Users::UpdateService.new(user, last_credential_check_a: Time.now).execute
+            Users::UpdateService.new(user, last_credential_check_at: Time.now).execute
 
             true
           else
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 38dc82493cf..f19b325a126 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -20,7 +20,8 @@ module Gitlab
           counts: {
             boards: Board.count,
             ci_builds: ::Ci::Build.count,
-            ci_pipelines: ::Ci::Pipeline.count,
+            ci_internal_pipelines: ::Ci::Pipeline.internal.count,
+            ci_external_pipelines: ::Ci::Pipeline.external.count,
             ci_runners: ::Ci::Runner.count,
             ci_triggers: ::Ci::Trigger.count,
             ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb
index dbfe0941e4d..841fb681435 100644
--- a/lib/gitlab/view/presenter/base.rb
+++ b/lib/gitlab/view/presenter/base.rb
@@ -15,6 +15,11 @@ module Gitlab
           super(user, action, overriden_subject || subject)
         end
 
+        # delegate all #can? queries to the subject
+        def declarative_policy_delegate
+          subject
+        end
+
         class_methods do
           def presenter?
             true