diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb | 82 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/create.rb | 17 | ||||
| -rw-r--r-- | lib/gitlab/file_detector.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 20 | ||||
| -rw-r--r-- | lib/gitlab/markup_helper.rb | 13 | ||||
| -rw-r--r-- | lib/gitlab/shell.rb | 17 | ||||
| -rw-r--r-- | lib/tasks/gitlab/check.rake | 139 |
7 files changed, 135 insertions, 155 deletions
diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb new file mode 100644 index 00000000000..35bfc381180 --- /dev/null +++ b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true +# +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateClusterKubernetesNamespaceTable + include Gitlab::Database::MigrationHelpers + + BATCH_SIZE = 1_000 + + module Migratable + class KubernetesNamespace < ActiveRecord::Base + self.table_name = 'clusters_kubernetes_namespaces' + end + + class ClusterProject < ActiveRecord::Base + include EachBatch + + self.table_name = 'cluster_projects' + + belongs_to :project + + def self.with_no_kubernetes_namespace + where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id)) + end + + def namespace + slug = "#{project.path}-#{project.id}".downcase + slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') + end + + def service_account + "#{namespace}-service-account" + end + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + end + + def perform + cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index| + sql_values = sql_values_for(cluster_projects_batch) + + insert_into_cluster_kubernetes_namespace(sql_values) + end + end + + private + + def cluster_projects_with_no_kubernetes_namespace + Migratable::ClusterProject.with_no_kubernetes_namespace + end + + def sql_values_for(cluster_projects) + cluster_projects.map do |cluster_project| + values_for_cluster_project(cluster_project) + end + end + + def values_for_cluster_project(cluster_project) + { + cluster_project_id: cluster_project.id, + cluster_id: cluster_project.cluster_id, + project_id: cluster_project.project_id, + namespace: cluster_project.namespace, + service_account_name: cluster_project.service_account, + created_at: 'NOW()', + updated_at: 'NOW()' + } + end + + def insert_into_cluster_kubernetes_namespace(rows) + Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, + rows, + disable_quote: [:created_at, :updated_at]) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index c882241ef6a..aa627bdb009 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -7,26 +7,11 @@ module Gitlab class Create < Chain::Base include Chain::Helpers - # rubocop: disable CodeReuse/ActiveRecord def perform! - ::Ci::Pipeline.transaction do - pipeline.save! - - ## - # Create environments before the pipeline starts. - # - pipeline.builds.each do |build| - if build.has_environment? - project.environments.find_or_create_by( - name: build.expanded_environment_name - ) - end - end - end + pipeline.save! rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") end - # rubocop: enable CodeReuse/ActiveRecord def break? !pipeline.persisted? diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 4d89ee5a669..d6338b09e3d 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -8,7 +8,7 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: %r{\Areadme[^/]*\z}i, + readme: %r{\A(readme|index)[^/]*\z}i, changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, license: %r{\A((un)?licen[sc]e|copying)(\.[^/]+)?\z}i, contributing: %r{\Acontributing[^/]*\z}i, diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index fcc92341c40..20cd257bb98 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -720,6 +720,26 @@ module Gitlab end end + # Fetch remote for repository + # + # remote - remote name + # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication + # forced - should we use --force flag? + # no_tags - should we use --no-tags flag? + # prune - should we use --prune flag? + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + wrapped_gitaly_errors do + gitaly_repository_client.fetch_remote( + remote, + ssh_auth: ssh_auth, + forced: forced, + no_tags: no_tags, + prune: prune, + timeout: GITLAB_PROJECTS_TIMEOUT + ) + end + end + def blob_at(sha, path) Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index 142b7d1a472..d419fa66e57 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -4,10 +4,11 @@ module Gitlab module MarkupHelper extend self - MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze - ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze - OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze + ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze + OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS + PLAIN_FILENAMES = %w[readme index].freeze # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -43,7 +44,7 @@ module Gitlab # # Returns boolean def plain?(filename) - extension(filename) == 'txt' || filename.casecmp('readme').zero? + extension(filename) == 'txt' || plain_filename?(filename) end def previewable?(filename) @@ -55,5 +56,9 @@ module Gitlab def extension(filename) File.extname(filename).downcase.delete('.') end + + def plain_filename?(filename) + PLAIN_FILENAMES.include?(filename.downcase) + end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 16c1edb2f11..c6a6fb9b5ce 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -108,23 +108,6 @@ module Gitlab success end - # Fetch remote for repository - # - # repository - an instance of Git::Repository - # remote - remote name - # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication - # forced - should we use --force flag? - # no_tags - should we use --no-tags flag? - # - # Ex. - # fetch_remote(my_repo, "upstream") - # - def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - wrapped_gitaly_errors do - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) - end - end - # Move repository reroutes to mv_directory which is an alias for # mv_namespace. Given the underlying implementation is a move action, # indescriminate of what the folders might be. diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index e5b5f3548e4..663bebfe71a 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,6 +1,7 @@ namespace :gitlab do desc 'GitLab | Check the configuration of GitLab and its environment' task check: %w{gitlab:gitlab_shell:check + gitlab:gitaly:check gitlab:sidekiq:check gitlab:incoming_email:check gitlab:ldap:check @@ -44,13 +45,7 @@ namespace :gitlab do start_checking "GitLab Shell" check_gitlab_shell - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - check_repo_base_exists - check_repo_base_is_not_symlink - check_repo_base_user_and_group - check_repo_base_permissions - check_repos_hooks_directory_is_link - end + check_repos_hooks_directory_is_link check_gitlab_shell_self_test finished_checking "GitLab Shell" @@ -59,116 +54,6 @@ namespace :gitlab do # Checks ######################## - def check_repo_base_exists - puts "Repo base directory exists?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - if File.exist?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_repo_base_is_not_symlink - puts "Repo storage directories are symlinks?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun - end - end - end - - def check_repo_base_permissions - puts "Repo paths access is drwxrws---?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_repo_base_user_and_group - gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user - puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - user_id = uid_for(gitlab_shell_ssh_user) - root_group_id = gid_for('root') - group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)] - if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid) - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - def check_repos_hooks_directory_is_link print "hooks directories in repos are links: ... " @@ -247,6 +132,26 @@ namespace :gitlab do end end + namespace :gitaly do + desc 'GitLab | Check the health of Gitaly' + task check: :gitlab_environment do + warn_user_is_not_gitlab + start_checking 'Gitaly' + + Gitlab::HealthChecks::GitalyCheck.readiness.each do |result| + print "#{result.labels[:shard]} ... " + + if result.success + puts 'OK'.color(:green) + else + puts "FAIL: #{result.message}".color(:red) + end + end + + finished_checking 'Gitaly' + end + end + namespace :sidekiq do desc "GitLab | Check the configuration of Sidekiq" task check: :gitlab_environment do |