Merge branch 'master' into menu-resize-hidemenu-resize-hide

15 files changed, 157 insertions, 43 deletions

diff --git a/config/initializers/0_post_deployment_migrations.rb b/config/initializers/0_post_deployment_migrations.rb
new file mode 100644
index 00000000000..0068a03d214
--- /dev/null
+++ b/config/initializers/0_post_deployment_migrations.rb
@@ -0,0 +1,12 @@
+# Post deployment migrations are included by default. This file must be loaded
+# before other initializers as Rails may otherwise memoize a list of migrations
+# excluding the post deployment migrations.
+unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS']
+  path = Rails.root.join('db', 'post_migrate').to_s
+
+  Rails.application.config.paths['db/migrate'] << path
+
+  # Rails memoizes migrations at certain points where it won't read the above
+  # path just yet. As such we must also update the following list of paths.
+  ActiveRecord::Migrator.migrations_paths << path
+end
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 195108b921b..9ddd1554811 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -186,6 +186,7 @@ Settings.gitlab['email_enabled'] ||= true if Settings.gitlab['email_enabled'].ni
 Settings.gitlab['email_from'] ||= ENV['GITLAB_EMAIL_FROM'] || "gitlab@#{Settings.gitlab.host}"
 Settings.gitlab['email_display_name'] ||= ENV['GITLAB_EMAIL_DISPLAY_NAME'] || 'GitLab'
 Settings.gitlab['email_reply_to'] ||= ENV['GITLAB_EMAIL_REPLY_TO'] || "noreply@#{Settings.gitlab.host}"
+Settings.gitlab['email_subject_suffix'] ||= ENV['GITLAB_EMAIL_SUBJECT_SUFFIX'] || ""
 Settings.gitlab['base_url']   ||= Settings.send(:build_base_gitlab_url)
 Settings.gitlab['url']        ||= Settings.send(:build_gitlab_url)
 Settings.gitlab['user']       ||= 'git'
@@ -214,6 +215,7 @@ Settings.gitlab.default_projects_features['visibility_level']   = Settings.send(
 Settings.gitlab['domain_whitelist'] ||= []
 Settings.gitlab['import_sources'] ||= %w[github bitbucket gitlab google_code fogbugz git gitlab_project]
 Settings.gitlab['trusted_proxies'] ||= []
+Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config', 'no_todos_messages.yml'))
 
 #
 # CI
@@ -303,6 +305,13 @@ Settings.cron_jobs['prune_old_events_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['prune_old_events_worker']['cron'] ||= '* */6 * * *'
 Settings.cron_jobs['prune_old_events_worker']['job_class'] = 'PruneOldEventsWorker'
 
+Settings.cron_jobs['trending_projects_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['trending_projects_worker']['cron'] = '0 1 * * *'
+Settings.cron_jobs['trending_projects_worker']['job_class'] = 'TrendingProjectsWorker'
+Settings.cron_jobs['remove_unreferenced_lfs_objects_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['cron'] ||= '20 0 * * *'
+Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'RemoveUnreferencedLfsObjectsWorker'
+
 #
 # GitLab Shell
 #
diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb
new file mode 100644
index 00000000000..ae2ca258df1
--- /dev/null
+++ b/config/initializers/7_redis.rb
@@ -0,0 +1,3 @@
+# Make sure we initialize a Redis connection pool before Sidekiq starts
+# multi-threaded execution.
+Gitlab::Redis.with { nil }
diff --git a/config/initializers/ar5_batching.rb b/config/initializers/ar5_batching.rb
new file mode 100644
index 00000000000..35e8b3808e2
--- /dev/null
+++ b/config/initializers/ar5_batching.rb
@@ -0,0 +1,41 @@
+# Port ActiveRecord::Relation#in_batches from ActiveRecord 5.
+# https://github.com/rails/rails/blob/ac027338e4a165273607dccee49a3d38bc836794/activerecord/lib/active_record/relation/batches.rb#L184
+# TODO: this can be removed once we're using AR5.
+raise "Vendored ActiveRecord 5 code! Delete #{__FILE__}!" if ActiveRecord::VERSION::MAJOR >= 5
+
+module ActiveRecord
+  module Batches
+    # Differences from upstream: enumerator support was removed, and custom
+    # order/limit clauses are ignored without a warning.
+    def in_batches(of: 1000, start: nil, finish: nil, load: false)
+      raise "Must provide a block" unless block_given?
+
+      relation = self.reorder(batch_order).limit(of)
+      relation = relation.where(arel_table[primary_key].gteq(start)) if start
+      relation = relation.where(arel_table[primary_key].lteq(finish)) if finish
+      batch_relation = relation
+
+      loop do
+        if load
+          records = batch_relation.records
+          ids = records.map(&:id)
+          yielded_relation = self.where(primary_key => ids)
+          yielded_relation.load_records(records)
+        else
+          ids = batch_relation.pluck(primary_key)
+          yielded_relation = self.where(primary_key => ids)
+        end
+
+        break if ids.empty?
+
+        primary_key_offset = ids.last
+        raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
+
+        yield yielded_relation
+
+        break if ids.length < of
+        batch_relation = relation.where(arel_table[primary_key].gt(primary_key_offset))
+      end
+    end
+  end
+end
diff --git a/config/initializers/ar_speed_up_migration_checking.rb b/config/initializers/ar_speed_up_migration_checking.rb
new file mode 100644
index 00000000000..1fe5defc01d
--- /dev/null
+++ b/config/initializers/ar_speed_up_migration_checking.rb
@@ -0,0 +1,18 @@
+if Rails.env.test?
+  require 'active_record/migration'
+
+  module ActiveRecord
+    class Migrator
+      class << self
+        alias_method :migrations_unmemoized, :migrations
+
+        # This method is called a large number of times per rspec example, and
+        # it reads + parses `db/migrate/*` each time. Memoizing it can save 0.5
+        # seconds per spec.
+        def migrations(paths)
+          @migrations ||= migrations_unmemoized(paths)
+        end
+      end
+    end
+  end
+end
diff --git a/config/initializers/attr_encrypted_no_db_connection.rb b/config/initializers/attr_encrypted_no_db_connection.rb
index c668864089b..e007666b852 100644
--- a/config/initializers/attr_encrypted_no_db_connection.rb
+++ b/config/initializers/attr_encrypted_no_db_connection.rb
@@ -1,20 +1,21 @@
 module AttrEncrypted
   module Adapters
     module ActiveRecord
-      def attribute_instance_methods_as_symbols_with_no_db_connection
-        # Use with_connection so the connection doesn't stay pinned to the thread.
-        connected = ::ActiveRecord::Base.connection_pool.with_connection(&:active?) rescue false
-        
-        if connected
-          # Call version from AttrEncrypted::Adapters::ActiveRecord
-          attribute_instance_methods_as_symbols_without_no_db_connection
-        else
-          # Call version from AttrEncrypted, i.e., `super` with regards to AttrEncrypted::Adapters::ActiveRecord
-          AttrEncrypted.instance_method(:attribute_instance_methods_as_symbols).bind(self).call
+      module DBConnectionQuerier
+        def attribute_instance_methods_as_symbols
+          # Use with_connection so the connection doesn't stay pinned to the thread.
+          connected = ::ActiveRecord::Base.connection_pool.with_connection(&:active?) rescue false
+
+          if connected
+            # Call version from AttrEncrypted::Adapters::ActiveRecord
+            super
+          else
+            # Call version from AttrEncrypted, i.e., `super` with regards to AttrEncrypted::Adapters::ActiveRecord
+            AttrEncrypted.instance_method(:attribute_instance_methods_as_symbols).bind(self).call
+          end
         end
       end
-
-      alias_method_chain :attribute_instance_methods_as_symbols, :no_db_connection
+      prepend DBConnectionQuerier
     end
   end
 end
diff --git a/config/initializers/connection_fix.rb b/config/initializers/connection_fix.rb
index d831a1838ed..d0b1444f607 100644
--- a/config/initializers/connection_fix.rb
+++ b/config/initializers/connection_fix.rb
@@ -20,7 +20,7 @@ if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter)
         execute_without_retry(*args)
       rescue ActiveRecord::StatementInvalid => e
         if e.message =~ /server has gone away/i
-          warn "Server timed out, retrying"
+          warn "Lost connection to MySQL server during query"
           reconnect!
           retry
         else
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index a0a8f88584c..a8afc36fc78 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -213,22 +213,9 @@ Devise.setup do |config|
   end
 
   if Gitlab::LDAP::Config.enabled?
-    Gitlab.config.ldap.servers.values.each do |server|
-      if server['allow_username_or_email_login']
-        email_stripping_proc = ->(name) {name.gsub(/@.*\z/, '')}
-      else
-        email_stripping_proc = ->(name) {name}
-      end
-
-      config.omniauth server['provider_name'],
-        host:     server['host'],
-        base:     server['base'],
-        uid:      server['uid'],
-        port:     server['port'],
-        method:   server['method'],
-        bind_dn:  server['bind_dn'],
-        password: server['password'],
-        name_proc: email_stripping_proc
+    Gitlab::LDAP::Config.providers.each do |provider|
+      ldap_config = Gitlab::LDAP::Config.new(provider)
+      config.omniauth(provider, ldap_config.omniauth_options)
     end
   end
 
@@ -254,6 +241,10 @@ Devise.setup do |config|
         end
       end
 
+      if provider['name'] == 'shibboleth'
+        provider['args'][:fail_with_empty_uid] = true
+      end
+
       # A Hash from the configuration will be passed as is.
       provider_arguments << provider['args'].symbolize_keys
     end
diff --git a/config/initializers/gitlab_shell_secret_token.rb b/config/initializers/gitlab_shell_secret_token.rb
index 7454c33c9dd..529dcdd4644 100644
--- a/config/initializers/gitlab_shell_secret_token.rb
+++ b/config/initializers/gitlab_shell_secret_token.rb
@@ -1 +1 @@
-Gitlab::Shell.new.generate_and_link_secret_token
+Gitlab::Shell.ensure_secret_token!
diff --git a/config/initializers/metrics.rb b/config/initializers/metrics.rb
index be22085b0df..3b8771543e4 100644
--- a/config/initializers/metrics.rb
+++ b/config/initializers/metrics.rb
@@ -67,6 +67,7 @@ if Gitlab::Metrics.enabled?
       ['app', 'finders']                    => ['app', 'finders'],
       ['app', 'mailers', 'emails']          => ['app', 'mailers'],
       ['app', 'services', '**']             => ['app', 'services'],
+      ['lib', 'gitlab', 'conflicts']        => ['lib'],
       ['lib', 'gitlab', 'diff']             => ['lib'],
       ['lib', 'gitlab', 'email', 'message'] => ['lib'],
       ['lib', 'gitlab', 'checks']           => ['lib']
diff --git a/config/initializers/postgresql_limit_fix.rb b/config/initializers/postgresql_limit_fix.rb
index 0cb3aaf4d24..4224d857e8a 100644
--- a/config/initializers/postgresql_limit_fix.rb
+++ b/config/initializers/postgresql_limit_fix.rb
@@ -1,5 +1,19 @@
 if defined?(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
   class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+    module LimitFilter
+      def add_column(table_name, column_name, type, options = {})
+        options.delete(:limit) if type == :text
+        super(table_name, column_name, type, options)
+      end
+
+      def change_column(table_name, column_name, type, options = {})
+        options.delete(:limit) if type == :text
+        super(table_name, column_name, type, options)
+      end
+    end
+
+    prepend ActiveRecord::ConnectionAdapters::PostgreSQLAdapter::LimitFilter
+
     class TableDefinition
       def text(*args)
         options = args.extract_options!
@@ -9,18 +23,5 @@ if defined?(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
         column_names.each { |name| column(name, type, options) }
       end
     end
-
-    def add_column_with_limit_filter(table_name, column_name, type, options = {})
-      options.delete(:limit) if type == :text
-      add_column_without_limit_filter(table_name, column_name, type, options)
-    end
-
-    def change_column_with_limit_filter(table_name, column_name, type, options = {})
-      options.delete(:limit) if type == :text
-      change_column_without_limit_filter(table_name, column_name, type, options)
-    end
-
-    alias_method_chain :add_column, :limit_filter
-    alias_method_chain :change_column, :limit_filter
   end
 end
diff --git a/config/initializers/rack_attack_logging.rb b/config/initializers/rack_attack_logging.rb
new file mode 100644
index 00000000000..8bb9ea29c33
--- /dev/null
+++ b/config/initializers/rack_attack_logging.rb
@@ -0,0 +1,7 @@
+# Adds logging for all Rack Attack blocks and throttling events.
+
+ActiveSupport::Notifications.subscribe('rack.attack') do |name, start, finish, request_id, req|
+  if [:throttle, :blacklist].include? req.env['rack.attack.match_type']
+    Rails.logger.info("Rack_Attack: #{req.env['rack.attack.match_type']} #{req.ip} #{req.request_method} #{req.fullpath}")
+  end
+end
diff --git a/config/initializers/routing_draw.rb b/config/initializers/routing_draw.rb
new file mode 100644
index 00000000000..25003cf0239
--- /dev/null
+++ b/config/initializers/routing_draw.rb
@@ -0,0 +1,7 @@
+# Adds draw method into Rails routing
+# It allows us to keep routing splitted into files
+class ActionDispatch::Routing::Mapper
+  def draw(routes_name)
+    instance_eval(File.read(Rails.root.join("config/routes/#{routes_name}.rb")))
+  end
+end
diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb
index 5892c1de024..4f30d1265c8 100644
--- a/config/initializers/sentry.rb
+++ b/config/initializers/sentry.rb
@@ -18,6 +18,8 @@ if Rails.env.production?
       
       # Sanitize fields based on those sanitized from Rails.
       config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s)
+      # Sanitize authentication headers
+      config.sanitize_http_headers = %w[Authorization Private-Token]
       config.tags = { program: Gitlab::Sentry.program_context }
     end
   end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index f7e714cd6bc..b87b31d9697 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -2,6 +2,9 @@
 redis_config_hash = Gitlab::Redis.params
 redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
 
+# Default is to retry 25 times with exponential backoff. That's too much.
+Sidekiq.default_worker_options = { retry: 3 }
+
 Sidekiq.configure_server do |config|
   config.redis = redis_config_hash
 
@@ -26,6 +29,8 @@ Sidekiq.configure_server do |config|
   end
   Sidekiq::Cron::Job.load_from_hash! cron_jobs
 
+  Gitlab::SidekiqThrottler.execute!
+
   # Database pool should be at least `sidekiq_concurrency` + 2
   # For more info, see: https://github.com/mperham/sidekiq/blob/master/4.0-Upgrade.md
   config = ActiveRecord::Base.configurations[Rails.env] ||
@@ -42,3 +47,19 @@ end
 Sidekiq.configure_client do |config|
   config.redis = redis_config_hash
 end
+
+# The Sidekiq client API always adds the queue to the Sidekiq queue
+# list, but mail_room and gitlab-shell do not. This is only necessary
+# for monitoring.
+config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s)
+
+begin
+  Sidekiq.redis do |conn|
+    conn.pipelined do
+      config[:queues].each do |queue|
+        conn.sadd('queues', queue[0])
+      end
+    end
+  end
+rescue Redis::BaseError, SocketError
+end