From 888821f9ffb56c6fdf762f28dd42cf3b7226652f Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Thu, 19 Nov 2015 19:22:46 +0100 Subject: Add support for batch download operation --- lib/gitlab/lfs/response.rb | 105 +++++++++++++++++++++++++++++++++------------ lib/gitlab/lfs/router.rb | 2 +- 2 files changed, 79 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index 4202c786466..ddadc07ebba 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -26,7 +26,7 @@ module Gitlab def render_download_object_response(oid) render_response_to_download do - if check_download_sendfile_header? && check_download_accept_header? + if check_download_sendfile_header? render_lfs_sendfile(oid) else render_not_found @@ -34,20 +34,15 @@ module Gitlab end end - def render_lfs_api_auth - render_response_to_push do - request_body = JSON.parse(@request.body.read) - return render_not_found if request_body.empty? || request_body['objects'].empty? - - response = build_response(request_body['objects']) - [ - 200, - { - "Content-Type" => "application/json; charset=utf-8", - "Cache-Control" => "private", - }, - [JSON.dump(response)] - ] + def render_batch_operation_response + request_body = JSON.parse(@request.body.read) + case request_body["operation"] + when "download" + render_batch_download(request_body) + when "upload" + render_batch_upload(request_body) + else + render_forbidden end end @@ -142,6 +137,38 @@ module Gitlab end end + def render_batch_upload(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_push do + response = build_upload_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + + def render_batch_download(body) + return render_not_found if body.empty? || body['objects'].nil? + + render_response_to_download do + response = build_download_batch_response(body['objects']) + [ + 200, + { + "Content-Type" => "application/json; charset=utf-8", + "Cache-Control" => "private", + }, + [JSON.dump(response)] + ] + end + end + def render_lfs_download_hypermedia(oid) return render_not_found unless oid.present? @@ -266,10 +293,16 @@ module Gitlab @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set end - def build_response(objects) + def build_upload_batch_response(objects) selected_objects = select_existing_objects(objects) - upload_hypermedia(objects, selected_objects) + upload_hypermedia_links(objects, selected_objects) + end + + def build_download_batch_response(objects) + selected_objects = select_existing_objects(objects) + + download_hypermedia_links(objects, selected_objects) end def download_hypermedia(oid) @@ -279,7 +312,6 @@ module Gitlab { 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}", 'header' => { - 'Accept' => "application/vnd.git-lfs+json; charset=utf-8", 'Authorization' => @env['HTTP_AUTHORIZATION'] }.compact } @@ -287,21 +319,40 @@ module Gitlab } end - def upload_hypermedia(all_objects, existing_objects) + def download_hypermedia_links(all_objects, existing_objects) all_objects.each do |object| - object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid']) + # generate links only for existing objects + next unless existing_objects.include?(object['oid']) + + object['_links'] = { + 'download' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } end { 'objects' => all_objects } end - def hypermedia_links(object) - { - "upload" => { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", - 'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] } - }.compact - } + def upload_hypermedia_links(all_objects, existing_objects) + all_objects.each do |object| + # generate links only for non-existing objects + next if existing_objects.include?(object['oid']) + + object['_links'] = { + 'upload' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } + } + end + + { 'objects' => all_objects } end end end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb index 4809e834984..2b3f2e8e958 100644 --- a/lib/gitlab/lfs/router.rb +++ b/lib/gitlab/lfs/router.rb @@ -48,7 +48,7 @@ module Gitlab # Check for Batch API if post_path[0].ends_with?("/info/lfs/objects/batch") - lfs.render_lfs_api_auth + lfs.render_batch_operation_response else nil end -- cgit v1.2.1 From 14d95b0529eacb8f42c5184fb71bff3f326d1670 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 20 Nov 2015 11:59:32 +0100 Subject: Part of tests done [ci skip] --- lib/gitlab/lfs/response.rb | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index ddadc07ebba..0371e1a7107 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -42,7 +42,7 @@ module Gitlab when "upload" render_batch_upload(request_body) else - render_forbidden + render_not_found end end @@ -322,16 +322,21 @@ module Gitlab def download_hypermedia_links(all_objects, existing_objects) all_objects.each do |object| # generate links only for existing objects - next unless existing_objects.include?(object['oid']) - - object['_links'] = { - 'download' => { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", - 'header' => { - 'Authorization' => @env['HTTP_AUTHORIZATION'] - }.compact + if existing_objects.include?(object['oid']) + object['actions'] = { + 'download' => { + 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}", + 'header' => { + 'Authorization' => @env['HTTP_AUTHORIZATION'] + }.compact + } } - } + else + object['error'] = { + 'code' => 404, + 'message' => "Object does not exist on the server or you don't have permissions to access it", + } + end end { 'objects' => all_objects } @@ -342,7 +347,7 @@ module Gitlab # generate links only for non-existing objects next if existing_objects.include?(object['oid']) - object['_links'] = { + object['actions'] = { 'upload' => { 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}", 'header' => { -- cgit v1.2.1 From b8d8292bef996a5d074b21a82d4faec0edb2e2bf Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 20 Nov 2015 12:21:41 +0100 Subject: Fix upload tests, reformat code and make rubocop happy --- lib/gitlab/lfs/response.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index 0371e1a7107..ada518f9e04 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -321,7 +321,6 @@ module Gitlab def download_hypermedia_links(all_objects, existing_objects) all_objects.each do |object| - # generate links only for existing objects if existing_objects.include?(object['oid']) object['actions'] = { 'download' => { @@ -344,7 +343,7 @@ module Gitlab def upload_hypermedia_links(all_objects, existing_objects) all_objects.each do |object| - # generate links only for non-existing objects + # generate actions only for non-existing objects next if existing_objects.include?(object['oid']) object['actions'] = { -- cgit v1.2.1 From dbc0be1b275e92af5432cf7c5cf44de3dc8c9ca5 Mon Sep 17 00:00:00 2001 From: Marin Jankovski Date: Fri, 20 Nov 2015 12:45:30 +0100 Subject: Error 501 when client is using deprecated API. --- lib/gitlab/lfs/response.rb | 60 ++++++++++------------------------------------ lib/gitlab/lfs/router.rb | 4 +++- 2 files changed, 15 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index ada518f9e04..aceef53ba60 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -10,20 +10,6 @@ module Gitlab @request = request end - # Return a response for a download request - # Can be a response to: - # Request from a user to get the file - # Request from gitlab-workhorse which file to serve to the user - def render_download_hypermedia_response(oid) - render_response_to_download do - if check_download_accept_header? - render_lfs_download_hypermedia(oid) - else - render_not_found - end - end - end - def render_download_object_response(oid) render_response_to_download do if check_download_sendfile_header? @@ -66,13 +52,24 @@ module Gitlab end end + def render_unsupported_deprecated_api + [ + 501, + { "Content-Type" => "application/json; charset=utf-8" }, + [JSON.dump({ + 'message' => 'Server supports batch API only, please update your Git LFS client to version 0.6.0 and up.', + 'documentation_url' => "#{Gitlab.config.gitlab.url}/help", + })] + ] + end + private def render_not_enabled [ 501, { - "Content-Type" => "application/vnd.git-lfs+json", + "Content-Type" => "application/json; charset=utf-8", }, [JSON.dump({ 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.', @@ -169,21 +166,6 @@ module Gitlab end end - def render_lfs_download_hypermedia(oid) - return render_not_found unless oid.present? - - lfs_object = object_for_download(oid) - if lfs_object - [ - 200, - { "Content-Type" => "application/vnd.git-lfs+json" }, - [JSON.dump(download_hypermedia(oid))] - ] - else - render_not_found - end - end - def render_lfs_upload_ok(oid, size, tmp_file) if store_file(oid, size, tmp_file) [ @@ -226,10 +208,6 @@ module Gitlab @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile" end - def check_download_accept_header? - @env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8" - end - def user_can_fetch? # Check user access against the project they used to initiate the pull @user.can?(:download_code, @origin_project) @@ -305,20 +283,6 @@ module Gitlab download_hypermedia_links(objects, selected_objects) end - def download_hypermedia(oid) - { - '_links' => { - 'download' => - { - 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}", - 'header' => { - 'Authorization' => @env['HTTP_AUTHORIZATION'] - }.compact - } - } - } - end - def download_hypermedia_links(all_objects, existing_objects) all_objects.each do |object| if existing_objects.include?(object['oid']) diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb index 2b3f2e8e958..78d02891102 100644 --- a/lib/gitlab/lfs/router.rb +++ b/lib/gitlab/lfs/router.rb @@ -34,7 +34,7 @@ module Gitlab case path_match[1] when "info/lfs" - lfs.render_download_hypermedia_response(oid) + lfs.render_unsupported_deprecated_api when "gitlab-lfs" lfs.render_download_object_response(oid) else @@ -49,6 +49,8 @@ module Gitlab # Check for Batch API if post_path[0].ends_with?("/info/lfs/objects/batch") lfs.render_batch_operation_response + elsif post_path[0].ends_with?("/info/lfs/objects") + lfs.render_unsupported_deprecated_api else nil end -- cgit v1.2.1