diff options
| -rw-r--r-- | changelogs/unreleased/usage-count.yml | 5 | ||||
| -rw-r--r-- | lib/gitlab/database/count.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/database/count/exact_count_strategy.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/usage_data.rb | 18 | ||||
| -rw-r--r-- | spec/lib/gitlab/database/count/exact_count_strategy_spec.rb | 6 | ||||
| -rw-r--r-- | spec/lib/gitlab/usage_data_spec.rb | 25 |
6 files changed, 52 insertions, 6 deletions
diff --git a/changelogs/unreleased/usage-count.yml b/changelogs/unreleased/usage-count.yml new file mode 100644 index 00000000000..efff2615ce4 --- /dev/null +++ b/changelogs/unreleased/usage-count.yml @@ -0,0 +1,5 @@ +--- +title: Use approximate count for big tables for usage statistics. +merge_request: +author: +type: fixed diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index c996d786909..f3d37ccd72a 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -40,7 +40,7 @@ module Gitlab if strategy.enabled? models_with_missing_counts = models - counts_by_model.keys - break if models_with_missing_counts.empty? + break counts_by_model if models_with_missing_counts.empty? counts = strategy.new(models_with_missing_counts).count diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index 0276fe2b54f..fa6951eda22 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -20,6 +20,8 @@ module Gitlab models.each_with_object({}) do |model, data| data[model] = model.count end + rescue *CONNECTION_ERRORS + {} end def self.enabled? diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index bfcc8efdc96..008e9cd1d24 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -2,6 +2,8 @@ module Gitlab class UsageData + APPROXIMATE_COUNT_MODELS = [Label, MergeRequest, Note, Todo].freeze + class << self def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } @@ -73,12 +75,9 @@ module Gitlab issues: count(Issue), keys: count(Key), label_lists: count(List.label), - labels: count(Label), lfs_objects: count(LfsObject), - merge_requests: count(MergeRequest), milestone_lists: count(List.milestone), milestones: count(Milestone), - notes: count(Note), pages_domains: count(PagesDomain), projects: count(Project), projects_imported_from_github: count(Project.where(import_type: 'github')), @@ -86,10 +85,9 @@ module Gitlab releases: count(Release), remote_mirrors: count(RemoteMirror), snippets: count(Snippet), - todos: count(Todo), uploads: count(Upload), web_hooks: count(WebHook) - }.merge(services_usage) + }.merge(services_usage).merge(approximate_counts) } end # rubocop: enable CodeReuse/ActiveRecord @@ -164,6 +162,16 @@ module Gitlab fallback end # rubocop: enable CodeReuse/ActiveRecord + + def approximate_counts + approx_counts = Gitlab::Database::Count.approximate_counts(APPROXIMATE_COUNT_MODELS) + + APPROXIMATE_COUNT_MODELS.each_with_object({}) do |model, result| + key = model.name.underscore.pluralize.to_sym + + result[key] = approx_counts[model] || -1 + end + end end end end diff --git a/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb b/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb index f518bb3dc3e..3991c737a26 100644 --- a/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/exact_count_strategy_spec.rb @@ -16,6 +16,12 @@ describe Gitlab::Database::Count::ExactCountStrategy do expect(subject).to eq({ Project => 3, Identity => 1 }) end + + it 'returns default value if count times out' do + allow(models.first).to receive(:count).and_raise(ActiveRecord::StatementInvalid.new('')) + + expect(subject).to eq({}) + end end describe '.enabled?' do diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index e2de612ff46..deb19fe1a4b 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -213,4 +213,29 @@ describe Gitlab::UsageData do expect(described_class.count(relation, fallback: 15)).to eq(15) end end + + describe '#approximate_counts' do + it 'gets approximate counts for selected models' do + create(:label) + + expect(Gitlab::Database::Count).to receive(:approximate_counts) + .with(described_class::APPROXIMATE_COUNT_MODELS).once.and_call_original + + counts = described_class.approximate_counts.values + + expect(counts.count).to eq(described_class::APPROXIMATE_COUNT_MODELS.count) + expect(counts.any? { |count| count < 0 }).to be_falsey + end + + it 'returns default values if counts can not be retrieved' do + described_class::APPROXIMATE_COUNT_MODELS.map do |model| + model.name.underscore.pluralize.to_sym + end + + expect(Gitlab::Database::Count).to receive(:approximate_counts) + .and_return({}) + + expect(described_class.approximate_counts.values.uniq).to eq([-1]) + end + end end |