From 4e0ad3ad20c9fce7e3622b061efc327580b748b1 Mon Sep 17 00:00:00 2001 From: Francisco Redondo Marchena Date: Fri, 8 Aug 2014 14:58:43 +0000 Subject: Downloading chunks --- scripts/migrate-chunks | 118 +++++++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 58 deletions(-) diff --git a/scripts/migrate-chunks b/scripts/migrate-chunks index 00de11fc..8309a0d0 100755 --- a/scripts/migrate-chunks +++ b/scripts/migrate-chunks @@ -29,6 +29,33 @@ for key in morphologies.iterkeys(): morphologies[key] = [m for m in morphs if m['kind'] == key] #print 'There are: %d %s' %(len(morphologies[key]), key) +# look for a chunk morph in the repo +# NOTE: The following reimplements part of morphlib's remote repo cache stuff +def parse_repo_alias(repo): + if verbose: + print 'Parsing repo-alias %s' % repo + domain, path = repo.split(':') + if domain == 'baserock': + repo = 'ssh://git@git.baserock.org/baserock/%s' % path + elif domain == 'upstream': + repo = 'ssh://git@git.baserock.org/delta/%s' % path + else: + raise Exception("I don't know how to parse the repo-alias \"%s\"" % repo) + return repo + +def make_request(path): + server_url = 'http://git.baserock.org:8080/' + url = urlparse.urljoin(server_url, '/1.0/%s' % path) + handle = urllib2.urlopen(url) + return handle.read() + +def quote(*args): + return tuple(urllib.quote(string) for string in args) + +def cat_file(repo, ref, filename): + return make_request('files?repo=%s&ref=%s&filename=%s' % + quote(repo, ref, filename)) + # organise the definitions repo definitions_repo = sb.get_git_directory_name(sb.root_repository_url) def move_morphs(morphs, kind): @@ -61,70 +88,45 @@ def move_morphs(morphs, kind): if kind == 'stratum': print 'Moving %s into subdirectory' % kind for morph in morphs: - # Add path to the build-depends morphologies - for build_depends in morph['build-depends']: - build_depends['morph'] = 'strata/' + build_depends['morph'] + '.morph' - # Add chunk path on the chunk's morph field per stratum - subpath = 'strata/' + morph['name'] - subdir = os.path.join(definitions_repo, subpath) - subprocess.call(['mkdir', '-p', subdir]) - for chunk in morph['chunks']: - chunk['morph'] = subpath + '/' + chunk['name'] + '.morph' - loader.save_to_file(morph.filename,morph) - new_location = os.path.join(subdir, morph.filename) - #subprocess.call(['git', 'mv', morph.filename, new_location]) - #morph.filename = new_location - #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into subdirectory' %kind]) - - # Download chunks morphologies defined on the stratum and - # add them to the directory tree. - sys.exit(0) - - #for morph in morphs: -# print '%s' %morph -# str_morph = str(morph) -# new_morph = loader.parse_morphology_text(str_morph, morph.filename) -# new_location = os.path.join(subdir, morph.filename.replace('-' + kind,'',1)) -# loader.save_to_file(new_location,new_morph) -# print '%s' % new_location -# sys.exit(0) + # Add chunk path on the chunk's morph field per stratum + stratum_path = 'strata/' + morph['name'] + subdir = os.path.join(definitions_repo, stratum_path) + subprocess.call(['mkdir', '-p', subdir]) + + # Download chunks morphologies defined on the stratum and + # add them to the directory tree. + for chunk in morph['chunks']: + print "%s" % chunk['morph'] + name = morphlib.util.sanitise_morphology_path(chunk['morph']) + chunk['morph'] = stratum_path + '/' \ + + morphlib.util.sanitise_morphology_path(chunk['morph']) + ref = chunk['ref'] + repo = parse_repo_alias(chunk['repo']) + print "%s %s %s" %(name, ref, repo) + try: + chunk_morph = cat_file(repo, ref, name) + new_chunk = loader.load_from_string(chunk_morph) + loader.save_to_file(chunk['morph'], new_chunk) + print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) + except urllib2.HTTPError, err: + if err.code == 404: + print "%s not found on %s" %(name, repo) + + # Add path to the build-depends morphologies + for build_depends in morph['build-depends']: + build_depends['morph'] = 'strata/' + build_depends['morph'] + '.morph' + loader.save_to_file(morph.filename,morph) + new_location = os.path.join(subdir, morph.filename) #subprocess.call(['git', 'mv', morph.filename, new_location]) #morph.filename = new_location - #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into subdirectory' % kind]m -# look for a chunk morph in the repo -# NOTE: The following reimplements part of morphlib's remote repo cache stuff -def parse_repo_alias(repo): - if verbose: - print 'Parsing repo-alias %s' % repo - domain, path = repo.split(':') - if domain == 'baserock': - repo = 'ssh://git@git.baserock.org/baserock/%s' % path - elif domain == 'upstream': - repo = 'ssh://git@git.baserock.org/delta/%s' % path - else: - raise Exception("I don't know how to parse the repo-alias \"%s\"" % repo) - return repo - -def make_request(path): - server_url = 'http://git.baserock.org:8080/' - url = urlparse.urljoin(server_url, '/1.0/%s' % path) - handle = urllib2.urlopen(url) - return handle.read() - -def quote(*args): - return tuple(urllib.quote(string) for string in args) - -def cat_file(repo, ref, filename): - return make_request('files?repo=%s&ref=%s&filename=%s' % - quote(repo, ref, filename)) - - + #subprocess.call(['git', 'commit', '--quiet', '-m', + # 'Move %s into subdirectory' %kind]) + # Move the morphologies to it directory for key in morphologies.iterkeys(): print "Moving %s....\n" %key if key == 'stratum': move_morphs(morphologies[key], key) +sys.exit(0) #move_morphs(chunks, 'chunks') #move_morphs(strata, 'strata') -- cgit v1.2.1