diff options
author | Francisco Redondo Marchena <francisco.marchena@codethink.co.uk> | 2014-08-08 16:46:10 +0000 |
---|---|---|
committer | Francisco Redondo Marchena <francisco.marchena@codethink.co.uk> | 2014-08-08 16:46:10 +0000 |
commit | bdec7cfa3a8a4f80496796bc640dbf64c9a61dad (patch) | |
tree | 7f1a7ff28c616f7edc1ef1b915d1e88f72e7b9f3 | |
parent | 4e0ad3ad20c9fce7e3622b061efc327580b748b1 (diff) | |
download | definitions-bdec7cfa3a8a4f80496796bc640dbf64c9a61dad.tar.gz |
WIP: Error when load Json and replace comments by description
-rwxr-xr-x | scripts/migrate-chunks | 177 |
1 files changed, 96 insertions, 81 deletions
diff --git a/scripts/migrate-chunks b/scripts/migrate-chunks index 8309a0d0..e9351037 100755 --- a/scripts/migrate-chunks +++ b/scripts/migrate-chunks @@ -95,38 +95,53 @@ def move_morphs(morphs, kind): # Download chunks morphologies defined on the stratum and # add them to the directory tree. - for chunk in morph['chunks']: - print "%s" % chunk['morph'] - name = morphlib.util.sanitise_morphology_path(chunk['morph']) - chunk['morph'] = stratum_path + '/' \ - + morphlib.util.sanitise_morphology_path(chunk['morph']) - ref = chunk['ref'] - repo = parse_repo_alias(chunk['repo']) - print "%s %s %s" %(name, ref, repo) - try: - chunk_morph = cat_file(repo, ref, name) - new_chunk = loader.load_from_string(chunk_morph) - loader.save_to_file(chunk['morph'], new_chunk) - print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) - except urllib2.HTTPError, err: - if err.code == 404: - print "%s not found on %s" %(name, repo) - - # Add path to the build-depends morphologies - for build_depends in morph['build-depends']: - build_depends['morph'] = 'strata/' + build_depends['morph'] + '.morph' - loader.save_to_file(morph.filename,morph) - new_location = os.path.join(subdir, morph.filename) - #subprocess.call(['git', 'mv', morph.filename, new_location]) - #morph.filename = new_location - #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into subdirectory' %kind]) + for chunk in morph['chunks']: + name = morphlib.util.sanitise_morphology_path(chunk['morph']) + chunk['morph'] = stratum_path + '/' \ + + morphlib.util.sanitise_morphology_path(chunk['morph']) + ref = chunk['ref'] + repo = parse_repo_alias(chunk['repo']) + print "%s %s %s" %(name, ref, repo) + try: + chunk_morph = cat_file(repo, ref, name) + new_chunk = loader.load_from_string(chunk_morph) + loader.save_to_file(chunk['morph'], new_chunk) + print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) + except urllib2.HTTPError as err: + # If there si no morphology in the repository we assume that the morphology + # system will be autodetected, so we don't have to create a new one + # unless we shut down the autodetecting system (fallback system). + if err.code == 404: + print "%s not found in %s" %(name, repo) + except morphlib.morphloader.InvalidFieldError as err: + print "ERROR: %s in chunk %s" % (err, chunk_morph) + if "comments" in err: + fixed_chunk = str(chunk_morph).replace("comments", "description") + m_fixed_chunk = loader.load_from_string(chunk['morph'],fixed_chunk) + loader.save_to_file(chunk['morph'], m_fixed_chunk) + print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) + except morphlib.morphloader.MorphologyNotYamlError as err: + print "ERROR: %s in chunk %s" % (err, chunk_morph) + new_chunk = json.loads(chunk_morph) + print "%s" % new_chunk + sys.exit(0) + loader.save_to_file(chunk['morph'], new_chunk) + print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) + + # Add path to the build-depends morphologies + for build_depends in morph['build-depends']: + build_depends['morph'] = 'strata/' + build_depends['morph'] + '.morph' + loader.save_to_file(morph.filename,morph) + new_location = os.path.join(subdir, morph.filename) + #subprocess.call(['git', 'mv', morph.filename, new_location]) + #morph.filename = new_location + #subprocess.call(['git', 'commit', '--quiet', '-m', + # 'Move %s into subdirectory' %kind]) # Move the morphologies to it directory for key in morphologies.iterkeys(): print "Moving %s....\n" %key if key == 'stratum': move_morphs(morphologies[key], key) -sys.exit(0) #move_morphs(chunks, 'chunks') #move_morphs(strata, 'strata') @@ -134,57 +149,57 @@ sys.exit(0) #move_morphs(clusters, 'clusters') sys.exit(0) -def ls_tree(repo, ref): - info = yaml.load(make_request('trees?repo=%s&ref=%s' % quote(repo, ref))) - return info['tree'].keys() - -# get a list of ALL the chunks -chunk_map = {} -for stratum in strata: - for spec in stratum['chunks']: - if spec['name'] not in chunk_map: - chunk_map[spec['name']] = (spec['repo'], spec['ref']) - - -new_chunks = [] -i = 0 -n = len(chunk_map) -for name, (repo, ref) in chunk_map.iteritems(): - i += 1 - if verbose: - print '[%d/%d] Checking chunk: %s' % (i, n, name) - repo = parse_repo_alias(repo) - morph = morphlib.util.sanitise_morphology_path(name) - if morph in ls_tree(repo, ref): - print '[%d/%d] Getting chunk morph: %s' % (i, n, name) - text = cat_file(repo, ref, morph) - # convert the morphology to yaml, because we should be using yaml now - try: - morph_dict = json.loads(text) - except ValueError: - morph_dict = yaml.load(text) - output = yaml.dump(morph_dict, - Dumper=morphlib.morphloader.OrderedDumper, - default_flow_style=False) - path = os.path.join(definitions_repo, morph) - with open(path, 'w') as f: - f.write(output) - new_chunks.append(name) - -# move the new chunks -for chunk in new_chunks: - morph = morphlib.util.sanitise_morphology_path(chunk) - path = os.path.join(definitions_repo, os.path.join('chunks', morph)) - subprocess.call(['mv', morph, path]) - #subprocess.call(['git', 'add', path]) -#subprocess.call(['git', 'commit', '-m', 'Add chunk morphologies']) - -# update the chunk specs in the strata -for stratum in strata: - for spec in stratum['chunks']: - if 'morph' in spec: - del spec['morph'] - if spec['name'] in new_chunks: - morph = morphlib.util.sanitise_morphology_path(spec['name']) - spec['morph'] = os.path.join('chunks', morph) - loader.save_to_file(stratum.filename, stratum) +#def ls_tree(repo, ref): +# info = yaml.load(make_request('trees?repo=%s&ref=%s' % quote(repo, ref))) +# return info['tree'].keys() +# +## get a list of ALL the chunks +#chunk_map = {} +#for stratum in strata: +# for spec in stratum['chunks']: +# if spec['name'] not in chunk_map: +# chunk_map[spec['name']] = (spec['repo'], spec['ref']) +# +# +#new_chunks = [] +#i = 0 +#n = len(chunk_map) +#for name, (repo, ref) in chunk_map.iteritems(): +# i += 1 +# if verbose: +# print '[%d/%d] Checking chunk: %s' % (i, n, name) +# repo = parse_repo_alias(repo) +# morph = morphlib.util.sanitise_morphology_path(name) +# if morph in ls_tree(repo, ref): +# print '[%d/%d] Getting chunk morph: %s' % (i, n, name) +# text = cat_file(repo, ref, morph) +# # convert the morphology to yaml, because we should be using yaml now +# try: +# morph_dict = json.loads(text) +# except ValueError: +# morph_dict = yaml.load(text) +# output = yaml.dump(morph_dict, +# Dumper=morphlib.morphloader.OrderedDumper, +# default_flow_style=False) +# path = os.path.join(definitions_repo, morph) +# with open(path, 'w') as f: +# f.write(output) +# new_chunks.append(name) +# +## move the new chunks +#for chunk in new_chunks: +# morph = morphlib.util.sanitise_morphology_path(chunk) +# path = os.path.join(definitions_repo, os.path.join('chunks', morph)) +# subprocess.call(['mv', morph, path]) +# #subprocess.call(['git', 'add', path]) +##subprocess.call(['git', 'commit', '-m', 'Add chunk morphologies']) +# +## update the chunk specs in the strata +#for stratum in strata: +# for spec in stratum['chunks']: +# if 'morph' in spec: +# del spec['morph'] +# if spec['name'] in new_chunks: +# morph = morphlib.util.sanitise_morphology_path(spec['name']) +# spec['morph'] = os.path.join('chunks', morph) +# loader.save_to_file(stratum.filename, stratum) |