Downloading chunks

1 files changed, 60 insertions, 58 deletions

diff --git a/scripts/migrate-chunks b/scripts/migrate-chunks
index 00de11fc..8309a0d0 100755
--- a/scripts/migrate-chunks
+++ b/scripts/migrate-chunks
@@ -29,6 +29,33 @@ for key in morphologies.iterkeys():
     morphologies[key] = [m for m in morphs if m['kind'] == key]
     #print 'There are: %d %s' %(len(morphologies[key]), key)
 
+# look for a chunk morph in the repo
+# NOTE: The following reimplements part of morphlib's remote repo cache stuff
+def parse_repo_alias(repo):
+    if verbose:
+        print 'Parsing repo-alias %s' % repo
+    domain, path = repo.split(':')
+    if domain == 'baserock':
+        repo = 'ssh://git@git.baserock.org/baserock/%s' % path
+    elif domain == 'upstream':
+        repo = 'ssh://git@git.baserock.org/delta/%s' % path
+    else:
+        raise Exception("I don't know how to parse the repo-alias \"%s\"" % repo)
+    return repo
+
+def make_request(path):
+    server_url = 'http://git.baserock.org:8080/'
+    url = urlparse.urljoin(server_url, '/1.0/%s' % path)
+    handle = urllib2.urlopen(url)
+    return handle.read()
+
+def quote(*args):
+    return tuple(urllib.quote(string) for string in args)
+
+def cat_file(repo, ref, filename):
+    return make_request('files?repo=%s&ref=%s&filename=%s' %
+                        quote(repo, ref, filename))
+
 # organise the definitions repo
 definitions_repo = sb.get_git_directory_name(sb.root_repository_url)
 def move_morphs(morphs, kind):
@@ -61,70 +88,45 @@ def move_morphs(morphs, kind):
     if kind == 'stratum':
         print 'Moving %s into subdirectory' % kind
         for morph in morphs:
-             # Add path to the build-depends morphologies
-             for build_depends in morph['build-depends']:
-                 build_depends['morph'] = 'strata/' + build_depends['morph'] + '.morph'
-	     # Add chunk path on the chunk's morph field per stratum
-	     subpath = 'strata/' + morph['name']
-             subdir = os.path.join(definitions_repo, subpath)
-             subprocess.call(['mkdir', '-p', subdir])
-             for chunk in morph['chunks']:
-                 chunk['morph'] = subpath + '/' + chunk['name'] + '.morph'
-	     loader.save_to_file(morph.filename,morph)
-             new_location = os.path.join(subdir, morph.filename)
-             #subprocess.call(['git', 'mv', morph.filename, new_location])
-             #morph.filename = new_location
-             #subprocess.call(['git', 'commit', '--quiet', '-m',
-             #                 'Move %s into subdirectory' %kind])
-
-	     # Download chunks morphologies defined on the stratum and
-	     # add them to the directory tree.
-	sys.exit(0)
-	 
-    #for morph in morphs:
-#	print '%s' %morph
-#	str_morph = str(morph)
-#	new_morph = loader.parse_morphology_text(str_morph, morph.filename)
-#        new_location = os.path.join(subdir, morph.filename.replace('-' + kind,'',1))
-#	loader.save_to_file(new_location,new_morph)
-#	print '%s' % new_location
-#       sys.exit(0)
+	    # Add chunk path on the chunk's morph field per stratum
+	    stratum_path = 'strata/' + morph['name']
+            subdir = os.path.join(definitions_repo, stratum_path)
+            subprocess.call(['mkdir', '-p', subdir])
+
+	    # Download chunks morphologies defined on the stratum and
+	    # add them to the directory tree.
+        for chunk in morph['chunks']:
+            print "%s" % chunk['morph']
+            name = morphlib.util.sanitise_morphology_path(chunk['morph'])
+            chunk['morph'] = stratum_path + '/' \
+			     + morphlib.util.sanitise_morphology_path(chunk['morph'])
+            ref = chunk['ref']
+            repo = parse_repo_alias(chunk['repo'])
+            print "%s %s %s" %(name, ref, repo)
+	    try:
+                chunk_morph = cat_file(repo, ref, name)
+	        new_chunk = loader.load_from_string(chunk_morph)
+	        loader.save_to_file(chunk['morph'], new_chunk)
+		print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
+	    except urllib2.HTTPError, err:
+		if err.code == 404:
+                    print "%s not found on %s" %(name, repo)
+
+        # Add path to the build-depends morphologies
+        for build_depends in morph['build-depends']:
+            build_depends['morph'] = 'strata/' + build_depends['morph'] + '.morph'
+	loader.save_to_file(morph.filename,morph)
+        new_location = os.path.join(subdir, morph.filename)
         #subprocess.call(['git', 'mv', morph.filename, new_location])
         #morph.filename = new_location
-    #subprocess.call(['git', 'commit', '--quiet', '-m',
-    #                 'Move %s into subdirectory' % kind]m
-# look for a chunk morph in the repo
-# NOTE: The following reimplements part of morphlib's remote repo cache stuff
-def parse_repo_alias(repo):
-    if verbose:
-        print 'Parsing repo-alias %s' % repo
-    domain, path = repo.split(':')
-    if domain == 'baserock':
-        repo = 'ssh://git@git.baserock.org/baserock/%s' % path
-    elif domain == 'upstream':
-        repo = 'ssh://git@git.baserock.org/delta/%s' % path
-    else:
-        raise Exception("I don't know how to parse the repo-alias \"%s\"" % repo)
-    return repo
-
-def make_request(path):
-    server_url = 'http://git.baserock.org:8080/'
-    url = urlparse.urljoin(server_url, '/1.0/%s' % path)
-    handle = urllib2.urlopen(url)
-    return handle.read()
-
-def quote(*args):
-    return tuple(urllib.quote(string) for string in args)
-
-def cat_file(repo, ref, filename):
-    return make_request('files?repo=%s&ref=%s&filename=%s' %
-                        quote(repo, ref, filename))
-
-
+        #subprocess.call(['git', 'commit', '--quiet', '-m',
+        #                 'Move %s into subdirectory' %kind])
+	 
 # Move the morphologies to it directory
 for key in morphologies.iterkeys():
     print "Moving %s....\n" %key
     if key == 'stratum': move_morphs(morphologies[key], key)
+sys.exit(0)
 
 #move_morphs(chunks, 'chunks')
 #move_morphs(strata, 'strata')