From 64be2046a655cced89ff7c7f5b289a68e105fad9 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 20 Apr 2018 09:26:34 +0000 Subject: Merge branch '39345-get-raw-archive' into 'master' Add an API endpoint to download git repository snapshots Closes #39345 See merge request gitlab-org/gitlab-ce!18173 --- GITALY_SERVER_VERSION | 2 +- changelogs/unreleased/39345-get-raw-archive.yml | 5 +++ doc/api/projects.md | 25 ++++++++++- lib/api/api.rb | 1 + lib/api/helpers/project_snapshots_helpers.rb | 25 +++++++++++ lib/api/project_snapshots.rb | 19 ++++++++ lib/gitlab/git/repository.rb | 4 ++ lib/gitlab/gitaly_client/repository_service.rb | 16 +++++++ lib/gitlab/workhorse.rb | 14 ++++++ .../gitaly_client/repository_service_spec.rb | 11 +++++ spec/lib/gitlab/workhorse_spec.rb | 22 ++++++++++ spec/requests/api/project_snapshots_spec.rb | 51 ++++++++++++++++++++++ 12 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 changelogs/unreleased/39345-get-raw-archive.yml create mode 100644 lib/api/helpers/project_snapshots_helpers.rb create mode 100644 lib/api/project_snapshots.rb create mode 100644 spec/requests/api/project_snapshots_spec.rb diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 5f8cbfdb7d7..483b7719418 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.95.0 +0.96.1 diff --git a/changelogs/unreleased/39345-get-raw-archive.yml b/changelogs/unreleased/39345-get-raw-archive.yml new file mode 100644 index 00000000000..fa894432d4c --- /dev/null +++ b/changelogs/unreleased/39345-get-raw-archive.yml @@ -0,0 +1,5 @@ +--- +title: Add an API endpoint to download git repository snapshots +merge_request: 18173 +author: +type: added diff --git a/doc/api/projects.md b/doc/api/projects.md index a0cb5aa0820..8c3fdf17979 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -1375,4 +1375,27 @@ Read more in the [Project Badges](project_badges.md) documentation. ## Issue and merge request description templates -The non-default [issue and merge request description templates](../user/project/description_templates.md) are managed inside the project's repository. So you can manage them via the API through the [Repositories API](repositories.md) and the [Repository Files API](repository_files.md). \ No newline at end of file +The non-default [issue and merge request description templates](../user/project/description_templates.md) are managed inside the project's repository. So you can manage them via the API through the [Repositories API](repositories.md) and the [Repository Files API](repository_files.md). + +## Download snapshot of a git repository + +> Introduced in GitLab 10.7 + +This endpoint may only be accessed by an administrative user. + +Download a snapshot of the project (or wiki, if requested) git repository. This +snapshot is always in uncompressed [tar](https://en.wikipedia.org/wiki/Tar_(computing)) +format. + +If a repository is corrupted to the point where `git clone` does not work, the +snapshot may allow some of the data to be retrieved. + +``` +GET /projects/:id/snapshot +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `wiki` | boolean | no | Whether to download the wiki, rather than project, repository | + diff --git a/lib/api/api.rb b/lib/api/api.rb index 073471b4c4d..5139e869c71 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -154,6 +154,7 @@ module API mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectMilestones + mount ::API::ProjectSnapshots mount ::API::ProjectSnippets mount ::API::ProtectedBranches mount ::API::Repositories diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb new file mode 100644 index 00000000000..94798a8cb51 --- /dev/null +++ b/lib/api/helpers/project_snapshots_helpers.rb @@ -0,0 +1,25 @@ +module API + module Helpers + module ProjectSnapshotsHelpers + def authorize_read_git_snapshot! + authenticated_with_full_private_access! + end + + def send_git_snapshot(repository) + header(*Gitlab::Workhorse.send_git_snapshot(repository)) + end + + def snapshot_project + user_project + end + + def snapshot_repository + if to_boolean(params[:wiki]) + snapshot_project.wiki.repository + else + snapshot_project.repository + end + end + end + end +end diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb new file mode 100644 index 00000000000..71005acc587 --- /dev/null +++ b/lib/api/project_snapshots.rb @@ -0,0 +1,19 @@ +module API + class ProjectSnapshots < Grape::API + helpers ::API::Helpers::ProjectSnapshotsHelpers + + before { authorize_read_git_snapshot! } + + resource :projects do + desc 'Download a (possibly inconsistent) snapshot of a repository' do + detail 'This feature was introduced in GitLab 10.7' + end + params do + optional :wiki, type: Boolean, desc: 'Set to true to receive the wiki repository' + end + get ':id/snapshot' do + send_git_snapshot(snapshot_repository) + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 45819743699..aafa9b19440 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1216,6 +1216,10 @@ module Gitlab true end + def create_from_snapshot(url, auth) + gitaly_repository_client.create_from_snapshot(url, auth) + end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) gitaly_migrate(:rebase) do |is_enabled| if is_enabled diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 6441065f5fe..bb50d1c1640 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -226,6 +226,22 @@ module Gitlab ) end + def create_from_snapshot(http_url, http_auth) + request = Gitaly::CreateRepositoryFromSnapshotRequest.new( + repository: @gitaly_repo, + http_url: http_url, + http_auth: http_auth + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_repository_from_snapshot, + request, + timeout: GitalyClient.default_timeout + ) + end + def write_ref(ref_path, ref, old_ref, shell) request = Gitaly::WriteRefRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 153cb2a8bb1..1f060de657d 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -81,6 +81,20 @@ module Gitlab ] end + def send_git_snapshot(repository) + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetSnapshotRequest' => Gitaly::GetSnapshotRequest.new( + repository: repository.gitaly_repository + ).to_json + } + + [ + SEND_DATA_HEADER, + "git-snapshot:#{encode(params)}" + ] + end + def send_git_diff(repository, diff_refs) params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { diff --git a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb index 21592688bf0..da9287ff691 100644 --- a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb @@ -145,4 +145,15 @@ describe Gitlab::GitalyClient::RepositoryService do client.calculate_checksum end end + + describe '#create_from_snapshot' do + it 'sends a create_repository_from_snapshot message' do + expect_any_instance_of(Gitaly::RepositoryService::Stub) + .to receive(:create_repository_from_snapshot) + .with(gitaly_request_with_path(storage_name, relative_path), kind_of(Hash)) + .and_return(double) + + client.create_from_snapshot('http://example.com?wiki=1', 'Custom xyz') + end + end end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index d64ea72e346..e732b089d44 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -482,4 +482,26 @@ describe Gitlab::Workhorse do }.deep_stringify_keys) end end + + describe '.send_git_snapshot' do + let(:url) { 'http://example.com' } + + subject(:request) { described_class.send_git_snapshot(repository) } + + it 'sets the header correctly' do + key, command, params = decode_workhorse_header(request) + + expect(key).to eq("Gitlab-Workhorse-Send-Data") + expect(command).to eq('git-snapshot') + expect(params).to eq( + 'GitalyServer' => { + 'address' => Gitlab::GitalyClient.address(project.repository_storage), + 'token' => Gitlab::GitalyClient.token(project.repository_storage) + }, + 'GetSnapshotRequest' => Gitaly::GetSnapshotRequest.new( + repository: repository.gitaly_repository + ).to_json + ) + end + end end diff --git a/spec/requests/api/project_snapshots_spec.rb b/spec/requests/api/project_snapshots_spec.rb new file mode 100644 index 00000000000..07a920f8d28 --- /dev/null +++ b/spec/requests/api/project_snapshots_spec.rb @@ -0,0 +1,51 @@ +require 'spec_helper' + +describe API::ProjectSnapshots do + include WorkhorseHelpers + + let(:project) { create(:project) } + let(:admin) { create(:admin) } + + describe 'GET /projects/:id/snapshot' do + def expect_snapshot_response_for(repository) + type, params = workhorse_send_data + + expect(type).to eq('git-snapshot') + expect(params).to eq( + 'GitalyServer' => { + 'address' => Gitlab::GitalyClient.address(repository.project.repository_storage), + 'token' => Gitlab::GitalyClient.token(repository.project.repository_storage) + }, + 'GetSnapshotRequest' => Gitaly::GetSnapshotRequest.new( + repository: repository.gitaly_repository + ).to_json + ) + end + + it 'returns authentication error as project owner' do + get api("/projects/#{project.id}/snapshot", project.owner) + + expect(response).to have_gitlab_http_status(403) + end + + it 'returns authentication error as unauthenticated user' do + get api("/projects/#{project.id}/snapshot", nil) + + expect(response).to have_gitlab_http_status(401) + end + + it 'requests project repository raw archive as administrator' do + get api("/projects/#{project.id}/snapshot", admin), wiki: '0' + + expect(response).to have_gitlab_http_status(200) + expect_snapshot_response_for(project.repository) + end + + it 'requests wiki repository raw archive as administrator' do + get api("/projects/#{project.id}/snapshot", admin), wiki: '1' + + expect(response).to have_gitlab_http_status(200) + expect_snapshot_response_for(project.wiki.repository) + end + end +end -- cgit v1.2.1