summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFrancisco Redondo Marchena <francisco.marchena@codethink.co.uk>2014-08-08 16:46:10 +0000
committerFrancisco Redondo Marchena <francisco.marchena@codethink.co.uk>2014-08-08 16:46:10 +0000
commitbdec7cfa3a8a4f80496796bc640dbf64c9a61dad (patch)
tree7f1a7ff28c616f7edc1ef1b915d1e88f72e7b9f3
parent4e0ad3ad20c9fce7e3622b061efc327580b748b1 (diff)
downloaddefinitions-bdec7cfa3a8a4f80496796bc640dbf64c9a61dad.tar.gz
WIP: Error when load Json and replace comments by description
-rwxr-xr-xscripts/migrate-chunks177
1 files changed, 96 insertions, 81 deletions
diff --git a/scripts/migrate-chunks b/scripts/migrate-chunks
index 8309a0d0..e9351037 100755
--- a/scripts/migrate-chunks
+++ b/scripts/migrate-chunks
@@ -95,38 +95,53 @@ def move_morphs(morphs, kind):
# Download chunks morphologies defined on the stratum and
# add them to the directory tree.
- for chunk in morph['chunks']:
- print "%s" % chunk['morph']
- name = morphlib.util.sanitise_morphology_path(chunk['morph'])
- chunk['morph'] = stratum_path + '/' \
- + morphlib.util.sanitise_morphology_path(chunk['morph'])
- ref = chunk['ref']
- repo = parse_repo_alias(chunk['repo'])
- print "%s %s %s" %(name, ref, repo)
- try:
- chunk_morph = cat_file(repo, ref, name)
- new_chunk = loader.load_from_string(chunk_morph)
- loader.save_to_file(chunk['morph'], new_chunk)
- print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
- except urllib2.HTTPError, err:
- if err.code == 404:
- print "%s not found on %s" %(name, repo)
-
- # Add path to the build-depends morphologies
- for build_depends in morph['build-depends']:
- build_depends['morph'] = 'strata/' + build_depends['morph'] + '.morph'
- loader.save_to_file(morph.filename,morph)
- new_location = os.path.join(subdir, morph.filename)
- #subprocess.call(['git', 'mv', morph.filename, new_location])
- #morph.filename = new_location
- #subprocess.call(['git', 'commit', '--quiet', '-m',
- # 'Move %s into subdirectory' %kind])
+ for chunk in morph['chunks']:
+ name = morphlib.util.sanitise_morphology_path(chunk['morph'])
+ chunk['morph'] = stratum_path + '/' \
+ + morphlib.util.sanitise_morphology_path(chunk['morph'])
+ ref = chunk['ref']
+ repo = parse_repo_alias(chunk['repo'])
+ print "%s %s %s" %(name, ref, repo)
+ try:
+ chunk_morph = cat_file(repo, ref, name)
+ new_chunk = loader.load_from_string(chunk_morph)
+ loader.save_to_file(chunk['morph'], new_chunk)
+ print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
+ except urllib2.HTTPError as err:
+ # If there si no morphology in the repository we assume that the morphology
+ # system will be autodetected, so we don't have to create a new one
+ # unless we shut down the autodetecting system (fallback system).
+ if err.code == 404:
+ print "%s not found in %s" %(name, repo)
+ except morphlib.morphloader.InvalidFieldError as err:
+ print "ERROR: %s in chunk %s" % (err, chunk_morph)
+ if "comments" in err:
+ fixed_chunk = str(chunk_morph).replace("comments", "description")
+ m_fixed_chunk = loader.load_from_string(chunk['morph'],fixed_chunk)
+ loader.save_to_file(chunk['morph'], m_fixed_chunk)
+ print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
+ except morphlib.morphloader.MorphologyNotYamlError as err:
+ print "ERROR: %s in chunk %s" % (err, chunk_morph)
+ new_chunk = json.loads(chunk_morph)
+ print "%s" % new_chunk
+ sys.exit(0)
+ loader.save_to_file(chunk['morph'], new_chunk)
+ print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
+
+ # Add path to the build-depends morphologies
+ for build_depends in morph['build-depends']:
+ build_depends['morph'] = 'strata/' + build_depends['morph'] + '.morph'
+ loader.save_to_file(morph.filename,morph)
+ new_location = os.path.join(subdir, morph.filename)
+ #subprocess.call(['git', 'mv', morph.filename, new_location])
+ #morph.filename = new_location
+ #subprocess.call(['git', 'commit', '--quiet', '-m',
+ # 'Move %s into subdirectory' %kind])
# Move the morphologies to it directory
for key in morphologies.iterkeys():
print "Moving %s....\n" %key
if key == 'stratum': move_morphs(morphologies[key], key)
-sys.exit(0)
#move_morphs(chunks, 'chunks')
#move_morphs(strata, 'strata')
@@ -134,57 +149,57 @@ sys.exit(0)
#move_morphs(clusters, 'clusters')
sys.exit(0)
-def ls_tree(repo, ref):
- info = yaml.load(make_request('trees?repo=%s&ref=%s' % quote(repo, ref)))
- return info['tree'].keys()
-
-# get a list of ALL the chunks
-chunk_map = {}
-for stratum in strata:
- for spec in stratum['chunks']:
- if spec['name'] not in chunk_map:
- chunk_map[spec['name']] = (spec['repo'], spec['ref'])
-
-
-new_chunks = []
-i = 0
-n = len(chunk_map)
-for name, (repo, ref) in chunk_map.iteritems():
- i += 1
- if verbose:
- print '[%d/%d] Checking chunk: %s' % (i, n, name)
- repo = parse_repo_alias(repo)
- morph = morphlib.util.sanitise_morphology_path(name)
- if morph in ls_tree(repo, ref):
- print '[%d/%d] Getting chunk morph: %s' % (i, n, name)
- text = cat_file(repo, ref, morph)
- # convert the morphology to yaml, because we should be using yaml now
- try:
- morph_dict = json.loads(text)
- except ValueError:
- morph_dict = yaml.load(text)
- output = yaml.dump(morph_dict,
- Dumper=morphlib.morphloader.OrderedDumper,
- default_flow_style=False)
- path = os.path.join(definitions_repo, morph)
- with open(path, 'w') as f:
- f.write(output)
- new_chunks.append(name)
-
-# move the new chunks
-for chunk in new_chunks:
- morph = morphlib.util.sanitise_morphology_path(chunk)
- path = os.path.join(definitions_repo, os.path.join('chunks', morph))
- subprocess.call(['mv', morph, path])
- #subprocess.call(['git', 'add', path])
-#subprocess.call(['git', 'commit', '-m', 'Add chunk morphologies'])
-
-# update the chunk specs in the strata
-for stratum in strata:
- for spec in stratum['chunks']:
- if 'morph' in spec:
- del spec['morph']
- if spec['name'] in new_chunks:
- morph = morphlib.util.sanitise_morphology_path(spec['name'])
- spec['morph'] = os.path.join('chunks', morph)
- loader.save_to_file(stratum.filename, stratum)
+#def ls_tree(repo, ref):
+# info = yaml.load(make_request('trees?repo=%s&ref=%s' % quote(repo, ref)))
+# return info['tree'].keys()
+#
+## get a list of ALL the chunks
+#chunk_map = {}
+#for stratum in strata:
+# for spec in stratum['chunks']:
+# if spec['name'] not in chunk_map:
+# chunk_map[spec['name']] = (spec['repo'], spec['ref'])
+#
+#
+#new_chunks = []
+#i = 0
+#n = len(chunk_map)
+#for name, (repo, ref) in chunk_map.iteritems():
+# i += 1
+# if verbose:
+# print '[%d/%d] Checking chunk: %s' % (i, n, name)
+# repo = parse_repo_alias(repo)
+# morph = morphlib.util.sanitise_morphology_path(name)
+# if morph in ls_tree(repo, ref):
+# print '[%d/%d] Getting chunk morph: %s' % (i, n, name)
+# text = cat_file(repo, ref, morph)
+# # convert the morphology to yaml, because we should be using yaml now
+# try:
+# morph_dict = json.loads(text)
+# except ValueError:
+# morph_dict = yaml.load(text)
+# output = yaml.dump(morph_dict,
+# Dumper=morphlib.morphloader.OrderedDumper,
+# default_flow_style=False)
+# path = os.path.join(definitions_repo, morph)
+# with open(path, 'w') as f:
+# f.write(output)
+# new_chunks.append(name)
+#
+## move the new chunks
+#for chunk in new_chunks:
+# morph = morphlib.util.sanitise_morphology_path(chunk)
+# path = os.path.join(definitions_repo, os.path.join('chunks', morph))
+# subprocess.call(['mv', morph, path])
+# #subprocess.call(['git', 'add', path])
+##subprocess.call(['git', 'commit', '-m', 'Add chunk morphologies'])
+#
+## update the chunk specs in the strata
+#for stratum in strata:
+# for spec in stratum['chunks']:
+# if 'morph' in spec:
+# del spec['morph']
+# if spec['name'] in new_chunks:
+# morph = morphlib.util.sanitise_morphology_path(spec['name'])
+# spec['morph'] = os.path.join('chunks', morph)
+# loader.save_to_file(stratum.filename, stratum)