diff options
| -rwxr-xr-x | scripts/migrate-chunks | 198 |
1 files changed, 99 insertions, 99 deletions
diff --git a/scripts/migrate-chunks b/scripts/migrate-chunks index 48bf5ee0..3c01d0c2 100755 --- a/scripts/migrate-chunks +++ b/scripts/migrate-chunks @@ -11,29 +11,22 @@ import urlparse import yaml import re -verbose = False -if '--verbose' in sys.argv or '-v' in sys.argv: - verbose = True - -# load all morphologies in the definitions repo +# Load all morphologies in the definitions repo sb = morphlib.sysbranchdir.open_from_within('.') loader = morphlib.morphloader.MorphologyLoader() morphs = [m for m in sb.load_all_morphologies(loader)] print 'Found %d morphologies in total' % len(morphs) -# Harvest all the morphologies on the directory +# Clasify the morphologies regarding of their kind field morphologies = { 'chunk': '', 'stratum': '', 'system' : '', 'cluster': '' } for key in morphologies.iterkeys(): morphologies[key] = [m for m in morphs if m['kind'] == key] #print 'There are: %d %s' %(len(morphologies[key]), key) -# look for a chunk morph in the repo # NOTE: The following reimplements part of morphlib's remote repo cache stuff def parse_repo_alias(repo): - if verbose: - print 'Parsing repo-alias %s' % repo domain, path = repo.split(':') if domain == 'baserock': repo = 'ssh://git@git.baserock.org/baserock/%s' % path @@ -57,16 +50,14 @@ def cat_file(repo, ref, filename): quote(repo, ref, filename)) def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'): - ''' This function pretends to replace the sanitise_morphology_path in - morphlib.utils. - This new funtion will receive the field kind to check/add the directory - tree. - Chunk morphologies should add the stratum which belongs to. + '''This function receives the name or the morph field of one morphology + and returns the path of the morphology depending on the name, kind and + if it belongs to other morphologies. ''' # Dictionary which match morphology's kind and morphology's # directory in definitions.git - morphology_path = { 'chunk': 'chunks', 'stratum': 'strata', - 'system':'systems', 'cluster': 'clusters'} + morph_dir = { 'chunk': 'chunks', 'stratum': 'strata', + 'system':'systems', 'cluster': 'clusters'} # For chunks morphologies we need to know to which stratums # belongs this chunk. if morph_kind == 'chunk': @@ -87,8 +78,8 @@ def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'): morph_kind = 'stratum' # Add the morphology path to the morph field. - if morphology_path[morph_kind] not in morph_field: - morph_field = morphology_path[morph_kind] + '/' + morph_field + if morph_dir[morph_kind] not in morph_field: + morph_field = morph_dir[morph_kind] + '/' + morph_field # Add the morphology suffix if the morphology. if not morph_field.endswith('.morph'): @@ -96,107 +87,114 @@ def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'): return morph_field -# organise the definitions repo -definitions_repo = sb.get_git_directory_name(sb.root_repository_url) -def move_clusters(morphs): - subdir = os.path.join(definitions_repo, 'clusters') - subprocess.call(['mkdir', '-p', subdir]) +def create_directory(name, path): + directory = os.path.join(path, name) + subprocess.call(['mkdir','-p', directory]) + return directory + +def move_morphologies(morphs, kind, directory, path): + # Dictionary which match morphology's kind and morphology's + # directory in definitions.git + morph_dir = { 'chunk': 'chunks', 'stratum': 'strata', + 'system':'systems', 'cluster': 'clusters'} + subdir = create_directory(directory, path) for morph in morphs: - for m in morph['systems']: - m['morph'] = sanitise_morphology_path(m['morph'], 'system') - print 'Moving %s into clusters' % morph.filename + for submorph in morph[morph_dir[kind]]: + submorph['morph'] = sanitise_morphology_path(submorph['morph'], kind) + print 'Moving %s into %s' % (morph.filename, directory) loader.save_to_file(morph.filename, morph) - new_location = os.path.join(subdir, morph.filename) + #new_location = os.path.join(subdir, morph.filename) #subprocess.call(['git', 'mv', morph.filename, new_location]) #morph.filename = new_location #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into clusters' % morph.filename]) + # 'Move %s into %s' %(morph['name'] + '.morph', directory)]) + +def move_clusters(morphs): + kind = 'system' + directory = 'clusters' + path = definitions_repo + move_morphologies(morphs, kind, directory, path) def move_systems(morphs): - subdir = os.path.join(definitions_repo, 'systems') - subprocess.call(['mkdir', '-p', subdir]) - for morph in morphs: - for stratum in morph['strata']: - stratum['morph'] = sanitise_morphology_path(stratum['morph'], 'stratum') - print 'Moving %s into systems' % morph.filename - loader.save_to_file(morph.filename,morph) - new_location = os.path.join(subdir, morph.filename) - #subprocess.call(['git', 'mv', morph.filename, new_location]) - #morph.filename = new_location - #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into systems' % morph.filename]) + kind = 'stratum' + directory = 'systems' + path = definitions_repo + move_morphologies(morphs, kind, directory, path) def move_chunks(morphs): # There are not spec for this yet print "No spec defined" - -def move_strata(morphs): - for morph in morphs: - # Add chunk path on the chunk's morph field per stratum - stratum_path = 'strata/' + morph['name'] - subdir = os.path.join(definitions_repo, stratum_path) - subprocess.call(['mkdir', '-p', subdir]) - - # Download chunks morphologies defined on the stratum and - # add them to the directory tree. - for chunk in morph['chunks']: - name = chunk['name'] + '.morph' - chunk['morph'] = sanitise_morphology_path(chunk['morph'],'chunk', morph['name']) - ref = chunk['ref'] - repo = parse_repo_alias(chunk['repo']) - try: - chunk_morph = cat_file(repo, ref, name) - new_chunk = loader.load_from_string(chunk_morph) - loader.save_to_file(chunk['morph'], new_chunk) - print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) - subprocess.call(['git', 'add', '--quiet', chunk['morph']) - #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into systems' % morph.filename]) - except urllib2.HTTPError as err: +def download_chunks(morph): + # Download chunks morphologies defined on the stratum and + # add them to the directory tree. + for chunk in morph['chunks']: + name = chunk['name'] + '.morph' + chunk['morph'] = sanitise_morphology_path(chunk['morph'],'chunk', morph['name']) + ref = chunk['ref'] + repo = parse_repo_alias(chunk['repo']) + try: + chunk_morph = cat_file(repo, ref, name) + new_chunk = loader.load_from_string(chunk_morph) + loader.save_to_file(chunk['morph'], new_chunk) + print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) + subprocess.call(['git', 'add', chunk['morph']]) + #subprocess.call(['git', 'commit', '--quiet', '-m', + # 'Add %s to definitions repository' % name]) + except urllib2.HTTPError as err: # If there is no morphology in the repository we assume that the morphology # system will be autodetected, so we don't have to create a new one # unless we shut down the autodetecting system (fallback system). - if err.code == 404: - print "%s not found in %s" %(name, repo) - except morphlib.morphloader.InvalidFieldError as err: - print "ERROR: %s in chunk \n%s" % (err, chunk_morph) - if "comments" in str(err): + if err.code == 404: + print "%s not found in %s" %(name, repo) + except morphlib.morphloader.InvalidFieldError as err: + print "ERROR: %s in chunk \n%s" % (err, chunk_morph) + if "comments" in str(err): # This error is caused because there are old morphologies which # contain the field "comments" instead of "description". # Replacing "comments" field by "description" will allow the morphology # to pass parse_morphology_text check and ready to be written to a file. - fixed_chunk = loader.parse_morphology_text(chunk_morph, name) - fixed_chunk['description'] = fixed_chunk.pop('comments') - loader.save_to_file(chunk['morph'], fixed_chunk) - print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) - subprocess.call(['git', 'add', '--quiet', chunk['morph']) - #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into systems' % morph.filename]) - if "buildsystem" in str(err): + fixed_chunk = loader.parse_morphology_text(chunk_morph, name) + fixed_chunk['description'] = fixed_chunk.pop('comments') + loader.save_to_file(chunk['morph'], fixed_chunk) + print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) + subprocess.call(['git', 'add', chunk['morph']]) + #subprocess.call(['git', 'commit', '--quiet', '-m', + # 'Add %s to definitions repository' % name]) + if "buildsystem" in str(err): # This error is caused because a typo in a morphology which # has a field "buildsystem" instead of "build-system". - fixed_chunk = loader.parse_morphology_text(chunk_morph, name) - fixed_chunk['build-system'] = fixed_chunk.pop('buildsystem') - loader.save_to_file(chunk['morph'], fixed_chunk) - print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) - subprocess.call(['git', 'add', '--quiet', chunk['morph']) - #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into systems' % morph.filename]) - except morphlib.morphloader.MorphologyNotYamlError as err: - print "ERROR: %s in chunk \n%s" % (err, chunk_morph) - # This error is caused because there are old morphologies written - # in JSON which contain '\t' characters. When try to load this - # kind of morphologies load_from_string fails when parse_morphology_text. - # Removing this characters will make load_from_string to load the morphology - # and translate it into a correct yaml format. - fix_chunk = chunk_morph.replace('\t','') - new_chunk = loader.load_from_string(fix_chunk) - loader.save_to_file(chunk['morph'], new_chunk) + fixed_chunk = loader.parse_morphology_text(chunk_morph, name) + fixed_chunk['build-system'] = fixed_chunk.pop('buildsystem') + loader.save_to_file(chunk['morph'], fixed_chunk) print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) - subprocess.call(['git', 'add', '--quiet', chunk['morph']) + subprocess.call(['git', 'add', chunk['morph']]) #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into systems' % morph.filename]) + # 'Add %s to definitions repository' % name]) + except morphlib.morphloader.MorphologyNotYamlError as err: + print "ERROR: %s in chunk \n%s" % (err, chunk_morph) + # This error is caused because there are old morphologies written + # in JSON which contain '\t' characters. When try to load this + # kind of morphologies load_from_string fails when parse_morphology_text. + # Removing this characters will make load_from_string to load the morphology + # and translate it into a correct yaml format. + fix_chunk = chunk_morph.replace('\t','') + new_chunk = loader.load_from_string(fix_chunk) + loader.save_to_file(chunk['morph'], new_chunk) + print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph']) + subprocess.call(['git', 'add', chunk['morph']]) + #subprocess.call(['git', 'commit', '--quiet', '-m', + # 'Add %s to definitions repository' % name]) + + +def move_strata(morphs): + for morph in morphs: + # Add chunk path on the chunk's morph field per stratum + stratum_path = 'strata/' + morph['name'] + subdir = create_directory(stratum_path, definitions_repo) + + # Download chunks which belongs to the stratum + download_chunks(morph) # Add path to the build-depends morphologies for build_depends in morph['build-depends']: @@ -206,9 +204,12 @@ def move_strata(morphs): #subprocess.call(['git', 'mv', morph.filename, new_location]) #morph.filename = new_location #subprocess.call(['git', 'commit', '--quiet', '-m', - # 'Move %s into subdirectory' %kind]) + # 'Move %s into strata' % morph.filename]) + +# Get the path from definitions repo +definitions_repo = sb.get_git_directory_name(sb.root_repository_url) -# Move the morphologies to it directory +# Move the morphologies to its directories for key in morphologies.iterkeys(): print "Moving %s....\n" %key if key == 'cluster': move_clusters(morphologies[key]) @@ -216,6 +217,5 @@ for key in morphologies.iterkeys(): elif key == 'stratum': move_strata(morphologies[key]) elif key == 'chunk': move_chunks(morphologies[key]) else: print 'ERROR: Morphology not know %' % key - #move_morphs(morphologies[key], key) sys.exit(0) |