From 577b9c9ad0d4c8560f8b93f8086e6509684a3816 Mon Sep 17 00:00:00 2001
From: Adam Coldrick <adam.coldrick@codethink.co.uk>
Date: Tue, 15 Jul 2014 12:54:18 +0000
Subject: Add a script to add chunk morphologies into definitions

---
 scripts/migrate-chunks | 130 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)
 create mode 100755 scripts/migrate-chunks

(limited to 'scripts/migrate-chunks')

diff --git a/scripts/migrate-chunks b/scripts/migrate-chunks
new file mode 100755
index 00000000..2e7ebc0d
--- /dev/null
+++ b/scripts/migrate-chunks
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+
+import json
+import morphlib
+import os
+import subprocess
+import sys
+import urllib
+import urllib2
+import urlparse
+import yaml
+
+verbose = False
+if '--verbose' in sys.argv or '-v' in sys.argv:
+    verbose = True
+
+# load all morphologies in the definitions repo
+sb = morphlib.sysbranchdir.open_from_within('.')
+loader = morphlib.morphloader.MorphologyLoader()
+morphs = [m for m in sb.load_all_morphologies(loader)]
+print 'Found %d morphologies in total' % len(morphs)
+
+# separate the morphologies by type
+chunks = [m for m in morphs if m['kind'] == 'chunk']
+print 'There are: %d chunks' % len(chunks)
+strata = [m for m in morphs if m['kind'] == 'stratum']
+print '           %d strata' % len(strata)
+systems = [m for m in morphs if m['kind'] == 'system']
+print '           %d systems' % len(systems)
+clusters = [m for m in morphs if m['kind'] == 'cluster']
+print '           %d clusters' % len(clusters)
+
+# organise the definitions repo
+definitions_repo = sb.get_git_directory_name(sb.root_repository_url)
+def move_morphs(morphs, kind):
+    print 'Moving %s into subdirectory' % kind
+    subdir = os.path.join(definitions_repo, kind)
+    subprocess.call(['mkdir', subdir])
+    for morph in morphs:
+        new_location = os.path.join(subdir, morph.filename)
+        subprocess.call(['git', 'mv', morph.filename, new_location])
+        morph.filename = new_location
+    subprocess.call(['git', 'commit', '--quiet', '-m',
+                     'Move %s into subdirectory' % kind])
+
+move_morphs(chunks, 'chunks')
+move_morphs(strata, 'strata')
+move_morphs(systems, 'systems')
+move_morphs(clusters, 'clusters')
+
+# get a list of ALL the chunks
+chunk_map = {}
+for stratum in strata:
+    for spec in stratum['chunks']:
+        if spec['name'] not in chunk_map:
+            chunk_map[spec['name']] = (spec['repo'], spec['ref'])
+
+# look for a chunk morph in the repo
+# NOTE: The following reimplements part of morphlib's remote repo cache stuff
+def parse_repo_alias(repo):
+    if verbose:
+        print 'Parsing repo-alias %s' % repo
+    domain, path = repo.split(':')
+    if domain == 'baserock':
+        repo = 'ssh://git@git.baserock.org/baserock/%s' % path
+    elif domain == 'upstream':
+        repo = 'ssh://git@git.baserock.org/delta/%s' % path
+    else:
+        raise Exception("I don't know how to parse the repo-alias \"%s\"" % repo)
+    return repo
+
+def make_request(path):
+    server_url = 'http://git.baserock.org:8080/'
+    url = urlparse.urljoin(server_url, '/1.0/%s' % path)
+    handle = urllib2.urlopen(url)
+    return handle.read()
+
+def quote(*args):
+    return tuple(urllib.quote(string) for string in args)
+
+def ls_tree(repo, ref):
+    info = yaml.load(make_request('trees?repo=%s&ref=%s' % quote(repo, ref)))
+    return info['tree'].keys()
+
+def cat_file(repo, ref, filename):
+    return make_request('files?repo=%s&ref=%s&filename=%s' %
+                        quote(repo, ref, filename))
+
+new_chunks = []
+i = 0
+n = len(chunk_map)
+for name, (repo, ref) in chunk_map.iteritems():
+    i += 1
+    if verbose:
+        print '[%d/%d] Checking chunk: %s' % (i, n, name)
+    repo = parse_repo_alias(repo)
+    morph = morphlib.util.sanitise_morphology_path(name)
+    if morph in ls_tree(repo, ref):
+        print '[%d/%d] Getting chunk morph: %s' % (i, n, name)
+        text = cat_file(repo, ref, morph)
+        # convert the morphology to yaml, because we should be using yaml now
+        try:
+            morph_dict = json.loads(text)
+        except ValueError:
+            morph_dict = yaml.load(text)
+        output = yaml.dump(morph_dict,
+                           Dumper=morphlib.morphloader.OrderedDumper,
+                           default_flow_style=False)
+        path = os.path.join(definitions_repo, morph)
+        with open(path, 'w') as f:
+            f.write(output)
+        new_chunks.append(name)
+
+# move the new chunks
+for chunk in new_chunks:
+    morph = morphlib.util.sanitise_morphology_path(chunk)
+    path = os.path.join(definitions_repo, os.path.join('chunks', morph))
+    subprocess.call(['mv', morph, path])
+    subprocess.call(['git', 'add', path])
+subprocess.call(['git', 'commit', '-m', 'Add chunk morphologies'])
+
+# update the chunk specs in the strata
+for stratum in strata:
+    for spec in stratum['chunks']:
+        if 'morph' in spec:
+            del spec['morph']
+        if spec['name'] in new_chunks:
+            morph = morphlib.util.sanitise_morphology_path(spec['name'])
+            spec['morph'] = os.path.join('chunks', morph)
+    loader.save_to_file(stratum.filename, stratum)
-- 
cgit v1.2.1