Merge branch 'config-yaml-differences' into 'master'

Backport changes made by EE to various YAML configuration files

See merge request gitlab-org/gitlab-ce!29606

10 files changed, 293 insertions, 11 deletions

diff --git a/config/database.yml.postgresql b/config/database.yml.postgresql
index baded682e46..5b3b35c9226 100644
--- a/config/database.yml.postgresql
+++ b/config/database.yml.postgresql
@@ -9,6 +9,15 @@ production:
   username: git
   password: "secure password"
   host: localhost
+  # load_balancing:
+  #   hosts:
+  #     - host1.example.com
+  #     - host2.example.com
+  #   discover:
+  #     nameserver: 1.2.3.4
+  #     port: 8600
+  #     record: secondary.postgresql.service.consul
+  #     interval: 300
 
 #
 # Development specific
diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml
index af76bace577..40a80429afa 100644
--- a/config/dependency_decisions.yml
+++ b/config/dependency_decisions.yml
@@ -599,3 +599,10 @@
     :why: https://github.com/apache/incubator-echarts/blob/master/LICENSE
     :versions: []
     :when: 2018-12-07 20:46:12.421256000 Z
+- - :license
+  - contracts
+  - BSD
+  - :who: Jarka Košanová
+    :why: https://github.com/egonSchiele/contracts.ruby/blob/master/LICENSE
+    :versions: []
+    :when: 2019-04-01 11:29:39.361015000 Z
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index c83f569d885..dddc5ec3540 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -230,7 +230,45 @@ production: &base
 
   ## Packages (maven repository, npm registry, etc...)
   packages:
-    enabled: false
+    enabled: true
+    # The location where build packages are stored (default: shared/packages).
+    # storage_path: shared/packages
+    object_store:
+      enabled: false
+      remote_directory: packages # The bucket name
+      # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false)
+      # background_upload: false # Temporary option to limit automatic upload (Default: true)
+      # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage
+      connection:
+        provider: AWS
+        aws_access_key_id: AWS_ACCESS_KEY_ID
+        aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+        region: us-east-1
+        # host: 'localhost' # default: s3.amazonaws.com
+        # endpoint: 'http://127.0.0.1:9000' # default: nil
+        # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4.
+        # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
+
+  ## Dependency Proxy
+  dependency_proxy:
+    enabled: true
+    # The location where build packages are stored (default: shared/dependency_proxy).
+    # storage_path: shared/dependency_proxy
+    object_store:
+      enabled: false
+      remote_directory: dependency_proxy # The bucket name
+      # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false)
+      # background_upload: false # Temporary option to limit automatic upload (Default: true)
+      # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage
+      connection:
+        provider: AWS
+        aws_access_key_id: AWS_ACCESS_KEY_ID
+        aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+        region: us-east-1
+        # host: 'localhost' # default: s3.amazonaws.com
+        # endpoint: 'http://127.0.0.1:9000' # default: nil
+        # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4.
+        # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
 
   ## GitLab Pages
   pages:
@@ -306,6 +344,53 @@ production: &base
     schedule_migrate_external_diffs_worker:
       cron: "15 * * * *"
 
+  # GitLab EE only jobs. These jobs are automatically enabled for an EE
+  # installation, and ignored for a CE installation.
+  ee_cron_jobs:
+    # Snapshot active users statistics
+    historical_data_worker:
+      cron: "0 12 * * *"
+
+    # In addition to refreshing users when they log in,
+    # periodically refresh LDAP users membership.
+    # NOTE: This will only take effect if LDAP is enabled
+    ldap_sync_worker:
+      cron: "30 1 * * *"
+
+    # Periodically refresh LDAP groups membership.
+    # NOTE: This will only take effect if LDAP is enabled
+    ldap_group_sync_worker:
+      cron: "0 * * * *"
+
+    # GitLab Geo metrics update worker
+    # NOTE: This will only take effect if Geo is enabled
+    geo_metrics_update_worker:
+      cron: "*/1 * * * *"
+
+    # GitLab Geo prune event log worker
+    # NOTE: This will only take effect if Geo is enabled (primary node only)
+    geo_prune_event_log_worker:
+      cron: "*/5 * * * *"
+
+    # GitLab Geo repository sync worker
+    # NOTE: This will only take effect if Geo is enabled (secondary nodes only)
+    geo_repository_sync_worker:
+      cron: "*/1 * * * *"
+
+    # GitLab Geo file download dispatch worker
+    # NOTE: This will only take effect if Geo is enabled (secondary nodes only)
+    geo_file_download_dispatch_worker:
+      cron: "*/1 * * * *"
+
+    # GitLab Geo migrated local files clean up worker
+    # NOTE: This will only take effect if Geo is enabled (secondary nodes only)
+    geo_migrated_local_files_clean_up_worker:
+      cron: "15 */6 * * *"
+
+    # Export pseudonymized data in CSV format for analysis
+    pseudonymizer_worker:
+      cron: "0 * * * *"
+
   registry:
     # enabled: true
     # host: registry.example.com
@@ -323,6 +408,16 @@ production: &base
     # clientside_dsn: https://<key>@sentry.io/<project>
     # environment: 'production' # e.g. development, staging, production
 
+  ## Geo
+  # NOTE: These settings will only take effect if Geo is enabled
+  geo:
+    # This is an optional identifier which Geo nodes can use to identify themselves.
+    # For example, if external_url is the same for two secondaries, you must specify
+    # a unique Geo node name for those secondaries.
+    #
+    # If it is blank, it defaults to external_url.
+    node_name: ''
+
   #
   # 2. GitLab CI settings
   # ==========================
@@ -349,6 +444,19 @@ production: &base
   #   bundle exec rake gitlab:ldap:check RAILS_ENV=production
   ldap:
     enabled: false
+
+    # This setting controls the number of seconds between LDAP permission checks
+    # for each user. After this time has expired for a given user, their next
+    # interaction with GitLab (a click in the web UI, a git pull, etc.) will be
+    # slower because the LDAP permission check is being performed. How much
+    # slower depends on your LDAP setup, but it is not uncommon for this check
+    # to add seconds of waiting time. The default value is to have a "slow
+    # click" once every 3600 seconds (i.e., once per hour).
+    #
+    # Warning: if you set this value too low, every click in GitLab will be a
+    # "slow click" for all of your LDAP users.
+    # sync_time: 3600
+
     servers:
       ##########################################################################
       #
@@ -446,6 +554,10 @@ production: &base
         # A value of 0 means there is no timeout.
         timeout: 10
 
+        # Enable smartcard authentication against the LDAP server. Valid values
+        # are "false", "optional", and "required".
+        smartcard_auth: false
+
         # This setting specifies if LDAP server is Active Directory LDAP server.
         # For non AD servers it skips the AD specific queries.
         # If your LDAP server is not AD, set this to false.
@@ -485,6 +597,31 @@ production: &base
         #
         user_filter: ''
 
+        # Base where we can search for groups
+        #
+        #   Ex. ou=Groups,dc=gitlab,dc=example
+        #
+        group_base: ''
+
+        # LDAP group of users who should be admins in GitLab
+        #
+        #   Ex. GLAdmins
+        #
+        admin_group: ''
+
+        # LDAP group of users who should be marked as external users in GitLab
+        #
+        #   Ex. ['Contractors', 'Interns']
+        #
+        external_groups: []
+
+        # Name of attribute which holds a ssh public key of the user object.
+        # If false or nil, SSH key syncronisation will be disabled.
+        #
+        #   Ex. sshpublickey
+        #
+        sync_ssh_keys: false
+
         # LDAP attributes that GitLab will use to create an account for the LDAP user.
         # The specified attribute can either be the attribute name as a string (e.g. 'mail'),
         # or an array of attribute names to try in order (e.g. ['mail', 'email']).
@@ -516,6 +653,38 @@ production: &base
       #   host:
       #   ....
 
+  ## Smartcard authentication settings
+  smartcard:
+    # Allow smartcard authentication
+    enabled: false
+
+    # Path to a file containing a CA certificate
+    ca_file: '/etc/ssl/certs/CA.pem'
+
+    # Port where the client side certificate is requested by the webserver (NGINX/Apache)
+    # client_certificate_required_port: 3444
+
+  ## Kerberos settings
+  kerberos:
+    # Allow the HTTP Negotiate authentication method for Git clients
+    enabled: false
+
+    # Kerberos 5 keytab file. The keytab file must be readable by the GitLab user,
+    # and should be different from other keytabs in the system.
+    # (default: use default keytab from Krb5 config)
+    # keytab: /etc/http.keytab
+
+    # The Kerberos service name to be used by GitLab.
+    # (default: accept any service name in keytab file)
+    # service_principal_name: HTTP/gitlab.example.com@EXAMPLE.COM
+
+    # Dedicated port: Git before 2.4 does not fall back to Basic authentication if Negotiate fails.
+    # To support both Basic and Negotiate methods with older versions of Git, configure
+    # nginx to proxy GitLab on an extra port (e.g. 8443) and uncomment the following lines
+    # to dedicate this port to Kerberos authentication. (default: false)
+    # use_dedicated_port: true
+    # port: 8443
+    # https: true
 
   ## OmniAuth settings
   omniauth:
@@ -635,6 +804,8 @@ production: &base
       #             name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient'
       #           } }
       #
+      # - { name: 'group_saml' }
+      #
       # - { name: 'crowd',
       #     args: {
       #       crowd_server_url: 'CROWD SERVER URL',
@@ -707,6 +878,20 @@ production: &base
     #   # Specifies Amazon S3 storage class to use for backups, this is optional
     #   # storage_class: 'STANDARD'
 
+  ## Pseudonymizer exporter
+  pseudonymizer:
+    # Tables manifest that specifies the fields to extract and pseudonymize.
+    manifest: config/pseudonymizer.yml
+    upload:
+      remote_directory: 'gitlab-elt'
+      # Fog storage connection settings, see http://fog.io/storage/ .
+      connection:
+      #   provider: AWS
+      #   region: eu-west-1
+      #   aws_access_key_id: AKIAKIAKI
+      #   aws_secret_access_key: 'secret123'
+      #   # The remote 'directory' to store the CSV files. For S3, this would be the bucket name.
+
   ## GitLab Shell settings
   gitlab_shell:
     path: /home/git/gitlab-shell/
@@ -872,6 +1057,17 @@ test:
     token: secret
   backup:
     path: tmp/tests/backups
+  pseudonymizer:
+    manifest: config/pseudonymizer.yml
+    upload:
+      # The remote 'directory' to store the CSV files. For S3, this would be the bucket name.
+      remote_directory: gitlab-elt.test
+      # Fog storage connection settings, see http://fog.io/storage/
+      connection:
+        provider: AWS # Only AWS supported at the moment
+        aws_access_key_id: AWS_ACCESS_KEY_ID
+        aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+        region: us-east-1
   gitlab_shell:
     path: tmp/tests/gitlab-shell/
     authorized_keys_file: tmp/tests/authorized_keys
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 0c8d94ccaed..6cca7a3b75f 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -117,6 +117,15 @@ if github_settings
     end
 end
 
+# SAML should be enabled for the tests automatically, but only for EE.
+saml_provider_enabled = Settings.omniauth.providers.any? do |provider|
+  provider['name'] == 'group_saml'
+end
+
+if Gitlab.ee? && Rails.env.test? && !saml_provider_enabled
+  Settings.omniauth.providers << Settingslogic.new({ 'name' => 'group_saml' })
+end
+
 Settings['shared'] ||= Settingslogic.new({})
 Settings.shared['path'] = Settings.absolute(Settings.shared['path'] || "shared")
 
@@ -291,6 +300,11 @@ Settings.gravatar['host']         = Settings.host_without_www(Settings.gravatar[
 # Cron Jobs
 #
 Settings['cron_jobs'] ||= Settingslogic.new({})
+
+if Gitlab.ee? && Settings['ee_cron_jobs']
+  Settings.cron_jobs.merge!(Settings.ee_cron_jobs)
+end
+
 Settings.cron_jobs['stuck_ci_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_ci_jobs_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['stuck_ci_jobs_worker']['job_class'] = 'StuckCiJobsWorker'
diff --git a/config/prometheus/common_metrics.yml b/config/prometheus/common_metrics.yml
index 3c67ca36791..f9ce5a6f365 100644
--- a/config/prometheus/common_metrics.yml
+++ b/config/prometheus/common_metrics.yml
@@ -171,7 +171,7 @@ panel_groups:
     weight: 2
     metrics:
     - id: system_metrics_kubernetes_container_memory_average
-      query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
+      query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
       label: Pod average
       unit: MB
   - title: "Canary: Memory Usage (Pod Average)"
@@ -190,7 +190,7 @@ panel_groups:
     weight: 1
     metrics:
     - id: system_metrics_kubernetes_container_core_usage
-      query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
+      query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
       label: Pod average
       unit: "cores"
   - title: "Canary: Core Usage (Pod Average)"
diff --git a/config/settings.rb b/config/settings.rb
index 1b94df785a7..6df2132332c 100644
--- a/config/settings.rb
+++ b/config/settings.rb
@@ -1,5 +1,11 @@
 require 'settingslogic'
 
+# We can not use `Rails.root` here, as this file might be loaded without the
+# full Rails environment being loaded. We can not use `require_relative` either,
+# as Rails uses `load` for `require_dependency` (used when loading the Rails
+# environment). This could then lead to this file being loaded twice.
+require_dependency File.expand_path('../lib/gitlab', __dir__)
+
 class Settings < Settingslogic
   source ENV.fetch('GITLAB_CONFIG') { Pathname.new(File.expand_path('..', __dir__)).join('config/gitlab.yml') }
   namespace ENV.fetch('GITLAB_ENV') { Rails.env }
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index fd9ce4d3374..4fda9d69077 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -93,3 +93,20 @@
   - [migrate_external_diffs, 1]
   - [update_project_statistics, 1]
   - [phabricator_import_import_tasks, 1]
+
+  # EE-specific queues
+  - [ldap_group_sync, 2]
+  - [create_github_webhook, 2]
+  - [geo, 1]
+  - [repository_update_mirror, 1]
+  - [new_epic, 2]
+  - [project_import_schedule, 1]
+  - [project_update_repository_storage, 1]
+  - [admin_emails, 1]
+  - [elastic_batch_project_indexer, 1]
+  - [elastic_indexer, 1]
+  - [elastic_full_index, 1]
+  - [elastic_commit_indexer, 1]
+  - [elastic_namespace_indexer, 1]
+  - [export_csv, 1]
+  - [incident_management, 2]
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index ccaf06c5d6a..fd4bbd69468 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_dependency 'gitlab/popen'
+require_dependency File.expand_path('gitlab/popen', __dir__)
 
 module Gitlab
   def self.root
@@ -60,11 +60,15 @@ module Gitlab
   end
 
   def self.ee?
-    if ENV['IS_GITLAB_EE'].present?
-      Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE'])
-    else
-      Object.const_defined?(:License)
-    end
+    @is_ee ||=
+      if ENV['IS_GITLAB_EE'].present?
+        Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE'])
+      else
+        # We may use this method when the Rails environment is not loaded. This
+        # means that checking the presence of the License class could result in
+        # this method returning `false`, even for an EE installation.
+        root.join('ee/app/models/license.rb').exist?
+      end
   end
 
   def self.http_proxy_env?
diff --git a/spec/fast_spec_helper.rb b/spec/fast_spec_helper.rb
index 0b5ab16ad71..91ef7653822 100644
--- a/spec/fast_spec_helper.rb
+++ b/spec/fast_spec_helper.rb
@@ -3,6 +3,7 @@ require 'bundler/setup'
 ENV['GITLAB_ENV'] = 'test'
 ENV['IN_MEMORY_APPLICATION_SETTINGS'] = 'true'
 
+require 'active_support/dependencies'
 require_relative '../config/settings'
 require_relative 'support/rspec'
 require 'active_support/all'
diff --git a/spec/lib/gitlab_spec.rb b/spec/lib/gitlab_spec.rb
index e075904b0cc..82b0e819063 100644
--- a/spec/lib/gitlab_spec.rb
+++ b/spec/lib/gitlab_spec.rb
@@ -97,14 +97,42 @@ describe Gitlab do
   end
 
   describe '.ee?' do
+    before do
+      described_class.instance_variable_set(:@is_ee, nil)
+    end
+
+    after do
+      described_class.instance_variable_set(:@is_ee, nil)
+    end
+
     it 'returns true when using Enterprise Edition' do
-      stub_const('License', Class.new)
+      root = Pathname.new('dummy')
+      license_path = double(:path, exist?: true)
+
+      allow(described_class)
+        .to receive(:root)
+        .and_return(root)
+
+      allow(root)
+        .to receive(:join)
+        .with('ee/app/models/license.rb')
+        .and_return(license_path)
 
       expect(described_class.ee?).to eq(true)
     end
 
     it 'returns false when using Community Edition' do
-      hide_const('License')
+      root = double(:path)
+      license_path = double(:path, exists?: false)
+
+      allow(described_class)
+        .to receive(:root)
+        .and_return(Pathname.new('dummy'))
+
+      allow(root)
+        .to receive(:join)
+        .with('ee/app/models/license.rb')
+        .and_return(license_path)
 
       expect(described_class.ee?).to eq(false)
     end