summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAdam Coldrick <adam.coldrick@codethink.co.uk>2014-07-15 12:54:18 +0000
committerAdam Coldrick <adam.coldrick@codethink.co.uk>2014-07-15 12:54:18 +0000
commit577b9c9ad0d4c8560f8b93f8086e6509684a3816 (patch)
treebbc01070a5f750119e3c7fc02681ffe219407552
parent39c18c5e0fe607a66d1f5a286034171e91383911 (diff)
downloaddefinitions-baserock/adamcoldrick/move-chunks-into-definitions.tar.gz
Add a script to add chunk morphologies into definitionsbaserock/adamcoldrick/move-chunks-into-definitions
-rwxr-xr-xscripts/migrate-chunks130
1 files changed, 130 insertions, 0 deletions
diff --git a/scripts/migrate-chunks b/scripts/migrate-chunks
new file mode 100755
index 00000000..2e7ebc0d
--- /dev/null
+++ b/scripts/migrate-chunks
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+
+import json
+import morphlib
+import os
+import subprocess
+import sys
+import urllib
+import urllib2
+import urlparse
+import yaml
+
+verbose = False
+if '--verbose' in sys.argv or '-v' in sys.argv:
+ verbose = True
+
+# load all morphologies in the definitions repo
+sb = morphlib.sysbranchdir.open_from_within('.')
+loader = morphlib.morphloader.MorphologyLoader()
+morphs = [m for m in sb.load_all_morphologies(loader)]
+print 'Found %d morphologies in total' % len(morphs)
+
+# separate the morphologies by type
+chunks = [m for m in morphs if m['kind'] == 'chunk']
+print 'There are: %d chunks' % len(chunks)
+strata = [m for m in morphs if m['kind'] == 'stratum']
+print ' %d strata' % len(strata)
+systems = [m for m in morphs if m['kind'] == 'system']
+print ' %d systems' % len(systems)
+clusters = [m for m in morphs if m['kind'] == 'cluster']
+print ' %d clusters' % len(clusters)
+
+# organise the definitions repo
+definitions_repo = sb.get_git_directory_name(sb.root_repository_url)
+def move_morphs(morphs, kind):
+ print 'Moving %s into subdirectory' % kind
+ subdir = os.path.join(definitions_repo, kind)
+ subprocess.call(['mkdir', subdir])
+ for morph in morphs:
+ new_location = os.path.join(subdir, morph.filename)
+ subprocess.call(['git', 'mv', morph.filename, new_location])
+ morph.filename = new_location
+ subprocess.call(['git', 'commit', '--quiet', '-m',
+ 'Move %s into subdirectory' % kind])
+
+move_morphs(chunks, 'chunks')
+move_morphs(strata, 'strata')
+move_morphs(systems, 'systems')
+move_morphs(clusters, 'clusters')
+
+# get a list of ALL the chunks
+chunk_map = {}
+for stratum in strata:
+ for spec in stratum['chunks']:
+ if spec['name'] not in chunk_map:
+ chunk_map[spec['name']] = (spec['repo'], spec['ref'])
+
+# look for a chunk morph in the repo
+# NOTE: The following reimplements part of morphlib's remote repo cache stuff
+def parse_repo_alias(repo):
+ if verbose:
+ print 'Parsing repo-alias %s' % repo
+ domain, path = repo.split(':')
+ if domain == 'baserock':
+ repo = 'ssh://git@git.baserock.org/baserock/%s' % path
+ elif domain == 'upstream':
+ repo = 'ssh://git@git.baserock.org/delta/%s' % path
+ else:
+ raise Exception("I don't know how to parse the repo-alias \"%s\"" % repo)
+ return repo
+
+def make_request(path):
+ server_url = 'http://git.baserock.org:8080/'
+ url = urlparse.urljoin(server_url, '/1.0/%s' % path)
+ handle = urllib2.urlopen(url)
+ return handle.read()
+
+def quote(*args):
+ return tuple(urllib.quote(string) for string in args)
+
+def ls_tree(repo, ref):
+ info = yaml.load(make_request('trees?repo=%s&ref=%s' % quote(repo, ref)))
+ return info['tree'].keys()
+
+def cat_file(repo, ref, filename):
+ return make_request('files?repo=%s&ref=%s&filename=%s' %
+ quote(repo, ref, filename))
+
+new_chunks = []
+i = 0
+n = len(chunk_map)
+for name, (repo, ref) in chunk_map.iteritems():
+ i += 1
+ if verbose:
+ print '[%d/%d] Checking chunk: %s' % (i, n, name)
+ repo = parse_repo_alias(repo)
+ morph = morphlib.util.sanitise_morphology_path(name)
+ if morph in ls_tree(repo, ref):
+ print '[%d/%d] Getting chunk morph: %s' % (i, n, name)
+ text = cat_file(repo, ref, morph)
+ # convert the morphology to yaml, because we should be using yaml now
+ try:
+ morph_dict = json.loads(text)
+ except ValueError:
+ morph_dict = yaml.load(text)
+ output = yaml.dump(morph_dict,
+ Dumper=morphlib.morphloader.OrderedDumper,
+ default_flow_style=False)
+ path = os.path.join(definitions_repo, morph)
+ with open(path, 'w') as f:
+ f.write(output)
+ new_chunks.append(name)
+
+# move the new chunks
+for chunk in new_chunks:
+ morph = morphlib.util.sanitise_morphology_path(chunk)
+ path = os.path.join(definitions_repo, os.path.join('chunks', morph))
+ subprocess.call(['mv', morph, path])
+ subprocess.call(['git', 'add', path])
+subprocess.call(['git', 'commit', '-m', 'Add chunk morphologies'])
+
+# update the chunk specs in the strata
+for stratum in strata:
+ for spec in stratum['chunks']:
+ if 'morph' in spec:
+ del spec['morph']
+ if spec['name'] in new_chunks:
+ morph = morphlib.util.sanitise_morphology_path(spec['name'])
+ spec['morph'] = os.path.join('chunks', morph)
+ loader.save_to_file(stratum.filename, stratum)