From affd049dc4427d749b97eaee37a5d54873016657 Mon Sep 17 00:00:00 2001 From: Alex Lossent Date: Mon, 27 Apr 2015 18:06:51 +0200 Subject: Improve handling of large diffs Diffs with a large number of changed lines time out (504 HTTP error) or generate a HTML page that's so heavy web browsers struggle with it. https://github.com/gitlabhq/gitlabhq/pull/5014 introduced limits on commit line count so that only a safe portion is rendered. This was later undone by code refactoring in be5b6db8, e0eb4803 and c741fcab. This patch re-introduces a safe limit on number of lines. --- CHANGELOG | 1 + app/helpers/diff_helper.rb | 19 +++++++--- app/views/projects/diffs/_diffs.html.haml | 8 +++-- app/views/projects/diffs/_warning.html.haml | 2 +- spec/helpers/diff_helper_spec.rb | 54 ++++++++++++++++++++++++++++- 5 files changed, 74 insertions(+), 10 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 88678b087c5..0995133d210 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -20,6 +20,7 @@ v 7.11.0 (unreleased) - Add "Reply quoting selected text" shortcut key (`r`) - Fix bug causing `@whatever` inside an issue's first code block to be picked up as a user mention. - Fix bug causing `@whatever` inside an inline code snippet (backtick-style) to be picked up as a user mention. + - Improve handling of large diffs - - Show Atom feed buttons everywhere where applicable. - Add project activity atom feed. diff --git a/app/helpers/diff_helper.rb b/app/helpers/diff_helper.rb index 162778ade58..1b10795bb7b 100644 --- a/app/helpers/diff_helper.rb +++ b/app/helpers/diff_helper.rb @@ -7,14 +7,23 @@ module DiffHelper end end - def safe_diff_files(diffs) - diffs.first(allowed_diff_size).map do |diff| - Gitlab::Diff::File.new(diff) + def allowed_diff_lines + if diff_hard_limit_enabled? + Commit::DIFF_HARD_LIMIT_LINES + else + Commit::DIFF_SAFE_LINES end end - def show_diff_size_warning?(diffs) - diffs.size > allowed_diff_size + def safe_diff_files(diffs) + lines = 0 + safe_files = [] + diffs.first(allowed_diff_size).each do |diff| + lines += diff.diff.lines.count + break if lines > allowed_diff_lines + safe_files << Gitlab::Diff::File.new(diff) + end + safe_files end def diff_hard_limit_enabled? diff --git a/app/views/projects/diffs/_diffs.html.haml b/app/views/projects/diffs/_diffs.html.haml index b49aee504fe..ec8974c5475 100644 --- a/app/views/projects/diffs/_diffs.html.haml +++ b/app/views/projects/diffs/_diffs.html.haml @@ -5,11 +5,13 @@ = parallel_diff_btn = render 'projects/diffs/stats', diffs: diffs -- if show_diff_size_warning?(diffs) - = render 'projects/diffs/warning', diffs: diffs +- diff_files = safe_diff_files(diffs) + +- if diff_files.count < diffs.size + = render 'projects/diffs/warning', diffs: diffs, shown_files_count: diff_files.count .files - - safe_diff_files(diffs).each_with_index do |diff_file, index| + - diff_files.each_with_index do |diff_file, index| = render 'projects/diffs/file', diff_file: diff_file, i: index, project: project - if @diff_timeout diff --git a/app/views/projects/diffs/_warning.html.haml b/app/views/projects/diffs/_warning.html.haml index 47abbba2eb2..bd0b7376ba7 100644 --- a/app/views/projects/diffs/_warning.html.haml +++ b/app/views/projects/diffs/_warning.html.haml @@ -14,6 +14,6 @@ = link_to "Email patch", merge_request_path(@merge_request, format: :patch), class: "btn btn-warning btn-sm" %p To preserve performance only - %strong #{allowed_diff_size} of #{diffs.size} + %strong #{shown_files_count} of #{diffs.size} files are displayed. diff --git a/spec/helpers/diff_helper_spec.rb b/spec/helpers/diff_helper_spec.rb index 95719b4b49f..dd4c1d645e2 100644 --- a/spec/helpers/diff_helper_spec.rb +++ b/spec/helpers/diff_helper_spec.rb @@ -5,7 +5,8 @@ describe DiffHelper do let(:project) { create(:project) } let(:commit) { project.commit(sample_commit.id) } - let(:diff) { commit.diffs.first } + let(:diffs) { commit.diffs } + let(:diff) { diffs.first } let(:diff_file) { Gitlab::Diff::File.new(diff) } describe 'diff_hard_limit_enabled?' do @@ -30,6 +31,57 @@ describe DiffHelper do end end + describe 'allowed_diff_lines' do + it 'should return hard limit for number of lines in a diff if force diff is true' do + allow(controller).to receive(:params) { { force_show_diff: true } } + expect(allowed_diff_lines).to eq(50000) + end + + it 'should return safe limit for numbers of lines a diff if force diff is false' do + expect(allowed_diff_lines).to eq(5000) + end + end + + describe 'safe_diff_files' do + it 'should return all files from a commit that is smaller than safe limits' do + expect(safe_diff_files(diffs).length).to eq(2) + end + + it 'should return only the first file if the diff line count in the 2nd file takes the total beyond safe limits' do + diffs[1].diff.stub(lines: [""] * 4999) #simulate 4999 lines + expect(safe_diff_files(diffs).length).to eq(1) + end + + it 'should return all files from a commit that is beyond safe limit for numbers of lines if force diff is true' do + allow(controller).to receive(:params) { { force_show_diff: true } } + diffs[1].diff.stub(lines: [""] * 4999) #simulate 4999 lines + expect(safe_diff_files(diffs).length).to eq(2) + end + + it 'should return only the first file if the diff line count in the 2nd file takes the total beyond hard limits' do + allow(controller).to receive(:params) { { force_show_diff: true } } + diffs[1].diff.stub(lines: [""] * 49999) #simulate 49999 lines + expect(safe_diff_files(diffs).length).to eq(1) + end + + it 'should return only a safe number of file diffs if a commit touches more files than the safe limits' do + large_diffs = diffs * 100 #simulate 200 diffs + expect(safe_diff_files(large_diffs).length).to eq(100) + end + + it 'should return all file diffs if a commit touches more files than the safe limits but force diff is true' do + allow(controller).to receive(:params) { { force_show_diff: true } } + large_diffs = diffs * 100 #simulate 200 diffs + expect(safe_diff_files(large_diffs).length).to eq(200) + end + + it 'should return a limited file diffs if a commit touches more files than the hard limits and force diff is true' do + allow(controller).to receive(:params) { { force_show_diff: true } } + very_large_diffs = diffs * 1000 #simulate 2000 diffs + expect(safe_diff_files(very_large_diffs).length).to eq(1000) + end + end + describe 'parallel_diff' do it 'should return an array of arrays containing the parsed diff' do expect(parallel_diff(diff_file, 0)). -- cgit v1.2.1 From 5ed7a39dc7aeff97bd7dceb9e125b352223e29a3 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 6 May 2015 12:25:23 -0400 Subject: Fix tooltips for event filter links --- app/helpers/events_helper.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/helpers/events_helper.rb b/app/helpers/events_helper.rb index c9fd0f0362b..18c75a8726b 100644 --- a/app/helpers/events_helper.rb +++ b/app/helpers/events_helper.rb @@ -25,12 +25,16 @@ module EventsHelper def event_filter_link(key, tooltip) key = key.to_s - active = if @event_filter.active? key - 'active' - end + active = 'active' if @event_filter.active?(key) + link_opts = { + class: 'event_filter_link', + id: "#{key}_event_filter", + title: "Filter by #{tooltip.downcase}", + data: { toggle: 'tooltip', placement: 'top' } + } content_tag :li, class: "filter_icon #{active}" do - link_to request.path, class: 'has_tooltip event_filter_link', id: "#{key}_event_filter", 'data-original-title' => 'Filter by ' + tooltip.downcase do + link_to request.path, link_opts do icon(icon_for_event[key]) + content_tag(:span, ' ' + tooltip) end end -- cgit v1.2.1 From 895bc33fef53b8ffd9eab0a9c31e3c81af8ec6a9 Mon Sep 17 00:00:00 2001 From: Job van der Voort Date: Thu, 7 May 2015 12:24:18 +0000 Subject: Update upgrader doc to point to 7.10.2 --- doc/update/upgrader.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/update/upgrader.md b/doc/update/upgrader.md index 2cca46c86a3..b2287083281 100644 --- a/doc/update/upgrader.md +++ b/doc/update/upgrader.md @@ -27,10 +27,10 @@ If you have local changes to your GitLab repository the script will stash them a Note: GitLab 7.9 adds `nodejs` as a dependency. GitLab 7.6 adds `libkrb5-dev` as a dependency (installed by default on Ubuntu and OSX). GitLab 7.2 adds `pkg-config` and `cmake` as dependency. Please check the dependencies in the [installation guide.](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/installation.md#1-packages-dependencies) cd /home/git/gitlab - sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.1"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' + sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.2"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' # to perform a non-interactive install (no user input required) you can add -y - # sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.1"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' -- -y + # sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.2"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' -- -y ## 3. Start application @@ -65,7 +65,7 @@ Here is a one line command with step 1 to 5 for the next time you upgrade: cd /home/git/gitlab; \ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production; \ sudo service gitlab stop; \ - sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.1"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' -- -y; \ + sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.2"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' -- -y; \ cd /home/git/gitlab-shell; \ sudo -u git -H git fetch; \ sudo -u git -H git checkout v`cat /home/git/gitlab/GITLAB_SHELL_VERSION`; \ @@ -73,4 +73,4 @@ cd /home/git/gitlab; \ sudo service gitlab start; \ sudo service nginx restart; \ sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production -``` +``` \ No newline at end of file -- cgit v1.2.1 From 93a2cc095bea039c92751673c4d81c04bab0b3a2 Mon Sep 17 00:00:00 2001 From: Sytse Sijbrandij Date: Thu, 7 May 2015 12:41:05 +0000 Subject: Fix spelling error, thanks Mark --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 82eac9cd0db..2e533ae9dd5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -94,7 +94,7 @@ sudo docker build --tag gitlab-data docker/data/ sudo docker build --tag gitlab-app:7.10.1 docker/app/ ``` -After this run the images as described in the prepivous section. +After this run the images as described in the previous section. We assume using a data volume container, this will simplify migrations and backups. This empty container will exist to persist as volumes the 3 directories used by GitLab, so remember not to delete it. @@ -152,4 +152,4 @@ sudo docker push sytse/gitlab_data ## Troubleshooting -Please see the [troubleshooting](troubleshooting.md) file in this directory. +Please see the [troubleshooting](troubleshooting.md) file in this directory. \ No newline at end of file -- cgit v1.2.1 From 415648e2555e25d30f64f4c2642cf149f6965859 Mon Sep 17 00:00:00 2001 From: Job van der Voort Date: Thu, 7 May 2015 13:39:10 +0000 Subject: revert to 7.10.1 --- doc/update/upgrader.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/update/upgrader.md b/doc/update/upgrader.md index b2287083281..d23534d58b6 100644 --- a/doc/update/upgrader.md +++ b/doc/update/upgrader.md @@ -27,10 +27,10 @@ If you have local changes to your GitLab repository the script will stash them a Note: GitLab 7.9 adds `nodejs` as a dependency. GitLab 7.6 adds `libkrb5-dev` as a dependency (installed by default on Ubuntu and OSX). GitLab 7.2 adds `pkg-config` and `cmake` as dependency. Please check the dependencies in the [installation guide.](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/installation.md#1-packages-dependencies) cd /home/git/gitlab - sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.2"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' + sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.1"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' # to perform a non-interactive install (no user input required) you can add -y - # sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.2"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' -- -y + # sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.1"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' -- -y ## 3. Start application @@ -65,7 +65,7 @@ Here is a one line command with step 1 to 5 for the next time you upgrade: cd /home/git/gitlab; \ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production; \ sudo service gitlab stop; \ - sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.2"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' -- -y; \ + sudo -u git -H ruby -Ilib -e 'require "gitlab/upgrader"' -e 'class Gitlab::Upgrader' -e 'def latest_version_raw' -e '"v7.10.1"' -e 'end' -e 'end' -e 'Gitlab::Upgrader.new.execute' -- -y; \ cd /home/git/gitlab-shell; \ sudo -u git -H git fetch; \ sudo -u git -H git checkout v`cat /home/git/gitlab/GITLAB_SHELL_VERSION`; \ -- cgit v1.2.1