summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFrancisco Redondo Marchena <francisco.marchena@codethink.co.uk>2014-08-12 16:50:43 (GMT)
committerFrancisco Redondo Marchena <francisco.marchena@codethink.co.uk>2014-08-12 16:50:43 (GMT)
commit3cbe2739ac37ba65cd7b995f9e87e8853587978b (patch)
tree996122a59f1fefbb0757564e4cb731452c115f3f
parent9ae937bd0582eef94c74617cb2928aaa7104bff0 (diff)
downloaddefinitions-3cbe2739ac37ba65cd7b995f9e87e8853587978b.tar.gz
WIP: Refactoring code in functions
-rwxr-xr-xscripts/migrate-chunks198
1 files changed, 99 insertions, 99 deletions
diff --git a/scripts/migrate-chunks b/scripts/migrate-chunks
index 48bf5ee..3c01d0c 100755
--- a/scripts/migrate-chunks
+++ b/scripts/migrate-chunks
@@ -11,29 +11,22 @@ import urlparse
import yaml
import re
-verbose = False
-if '--verbose' in sys.argv or '-v' in sys.argv:
- verbose = True
-
-# load all morphologies in the definitions repo
+# Load all morphologies in the definitions repo
sb = morphlib.sysbranchdir.open_from_within('.')
loader = morphlib.morphloader.MorphologyLoader()
morphs = [m for m in sb.load_all_morphologies(loader)]
print 'Found %d morphologies in total' % len(morphs)
-# Harvest all the morphologies on the directory
+# Clasify the morphologies regarding of their kind field
morphologies = { 'chunk': '', 'stratum': '', 'system' : '', 'cluster': '' }
for key in morphologies.iterkeys():
morphologies[key] = [m for m in morphs if m['kind'] == key]
#print 'There are: %d %s' %(len(morphologies[key]), key)
-# look for a chunk morph in the repo
# NOTE: The following reimplements part of morphlib's remote repo cache stuff
def parse_repo_alias(repo):
- if verbose:
- print 'Parsing repo-alias %s' % repo
domain, path = repo.split(':')
if domain == 'baserock':
repo = 'ssh://git@git.baserock.org/baserock/%s' % path
@@ -57,16 +50,14 @@ def cat_file(repo, ref, filename):
quote(repo, ref, filename))
def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'):
- ''' This function pretends to replace the sanitise_morphology_path in
- morphlib.utils.
- This new funtion will receive the field kind to check/add the directory
- tree.
- Chunk morphologies should add the stratum which belongs to.
+ '''This function receives the name or the morph field of one morphology
+ and returns the path of the morphology depending on the name, kind and
+ if it belongs to other morphologies.
'''
# Dictionary which match morphology's kind and morphology's
# directory in definitions.git
- morphology_path = { 'chunk': 'chunks', 'stratum': 'strata',
- 'system':'systems', 'cluster': 'clusters'}
+ morph_dir = { 'chunk': 'chunks', 'stratum': 'strata',
+ 'system':'systems', 'cluster': 'clusters'}
# For chunks morphologies we need to know to which stratums
# belongs this chunk.
if morph_kind == 'chunk':
@@ -87,8 +78,8 @@ def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'):
morph_kind = 'stratum'
# Add the morphology path to the morph field.
- if morphology_path[morph_kind] not in morph_field:
- morph_field = morphology_path[morph_kind] + '/' + morph_field
+ if morph_dir[morph_kind] not in morph_field:
+ morph_field = morph_dir[morph_kind] + '/' + morph_field
# Add the morphology suffix if the morphology.
if not morph_field.endswith('.morph'):
@@ -96,107 +87,114 @@ def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'):
return morph_field
-# organise the definitions repo
-definitions_repo = sb.get_git_directory_name(sb.root_repository_url)
-def move_clusters(morphs):
- subdir = os.path.join(definitions_repo, 'clusters')
- subprocess.call(['mkdir', '-p', subdir])
+def create_directory(name, path):
+ directory = os.path.join(path, name)
+ subprocess.call(['mkdir','-p', directory])
+ return directory
+
+def move_morphologies(morphs, kind, directory, path):
+ # Dictionary which match morphology's kind and morphology's
+ # directory in definitions.git
+ morph_dir = { 'chunk': 'chunks', 'stratum': 'strata',
+ 'system':'systems', 'cluster': 'clusters'}
+ subdir = create_directory(directory, path)
for morph in morphs:
- for m in morph['systems']:
- m['morph'] = sanitise_morphology_path(m['morph'], 'system')
- print 'Moving %s into clusters' % morph.filename
+ for submorph in morph[morph_dir[kind]]:
+ submorph['morph'] = sanitise_morphology_path(submorph['morph'], kind)
+ print 'Moving %s into %s' % (morph.filename, directory)
loader.save_to_file(morph.filename, morph)
- new_location = os.path.join(subdir, morph.filename)
+ #new_location = os.path.join(subdir, morph.filename)
#subprocess.call(['git', 'mv', morph.filename, new_location])
#morph.filename = new_location
#subprocess.call(['git', 'commit', '--quiet', '-m',
- # 'Move %s into clusters' % morph.filename])
+ # 'Move %s into %s' %(morph['name'] + '.morph', directory)])
+
+def move_clusters(morphs):
+ kind = 'system'
+ directory = 'clusters'
+ path = definitions_repo
+ move_morphologies(morphs, kind, directory, path)
def move_systems(morphs):
- subdir = os.path.join(definitions_repo, 'systems')
- subprocess.call(['mkdir', '-p', subdir])
- for morph in morphs:
- for stratum in morph['strata']:
- stratum['morph'] = sanitise_morphology_path(stratum['morph'], 'stratum')
- print 'Moving %s into systems' % morph.filename
- loader.save_to_file(morph.filename,morph)
- new_location = os.path.join(subdir, morph.filename)
- #subprocess.call(['git', 'mv', morph.filename, new_location])
- #morph.filename = new_location
- #subprocess.call(['git', 'commit', '--quiet', '-m',
- # 'Move %s into systems' % morph.filename])
+ kind = 'stratum'
+ directory = 'systems'
+ path = definitions_repo
+ move_morphologies(morphs, kind, directory, path)
def move_chunks(morphs):
# There are not spec for this yet
print "No spec defined"
-
-def move_strata(morphs):
- for morph in morphs:
- # Add chunk path on the chunk's morph field per stratum
- stratum_path = 'strata/' + morph['name']
- subdir = os.path.join(definitions_repo, stratum_path)
- subprocess.call(['mkdir', '-p', subdir])
-
- # Download chunks morphologies defined on the stratum and
- # add them to the directory tree.
- for chunk in morph['chunks']:
- name = chunk['name'] + '.morph'
- chunk['morph'] = sanitise_morphology_path(chunk['morph'],'chunk', morph['name'])
- ref = chunk['ref']
- repo = parse_repo_alias(chunk['repo'])
- try:
- chunk_morph = cat_file(repo, ref, name)
- new_chunk = loader.load_from_string(chunk_morph)
- loader.save_to_file(chunk['morph'], new_chunk)
- print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
- subprocess.call(['git', 'add', '--quiet', chunk['morph'])
- #subprocess.call(['git', 'commit', '--quiet', '-m',
- # 'Move %s into systems' % morph.filename])
- except urllib2.HTTPError as err:
+def download_chunks(morph):
+ # Download chunks morphologies defined on the stratum and
+ # add them to the directory tree.
+ for chunk in morph['chunks']:
+ name = chunk['name'] + '.morph'
+ chunk['morph'] = sanitise_morphology_path(chunk['morph'],'chunk', morph['name'])
+ ref = chunk['ref']
+ repo = parse_repo_alias(chunk['repo'])
+ try:
+ chunk_morph = cat_file(repo, ref, name)
+ new_chunk = loader.load_from_string(chunk_morph)
+ loader.save_to_file(chunk['morph'], new_chunk)
+ print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
+ subprocess.call(['git', 'add', chunk['morph']])
+ #subprocess.call(['git', 'commit', '--quiet', '-m',
+ # 'Add %s to definitions repository' % name])
+ except urllib2.HTTPError as err:
# If there is no morphology in the repository we assume that the morphology
# system will be autodetected, so we don't have to create a new one
# unless we shut down the autodetecting system (fallback system).
- if err.code == 404:
- print "%s not found in %s" %(name, repo)
- except morphlib.morphloader.InvalidFieldError as err:
- print "ERROR: %s in chunk \n%s" % (err, chunk_morph)
- if "comments" in str(err):
+ if err.code == 404:
+ print "%s not found in %s" %(name, repo)
+ except morphlib.morphloader.InvalidFieldError as err:
+ print "ERROR: %s in chunk \n%s" % (err, chunk_morph)
+ if "comments" in str(err):
# This error is caused because there are old morphologies which
# contain the field "comments" instead of "description".
# Replacing "comments" field by "description" will allow the morphology
# to pass parse_morphology_text check and ready to be written to a file.
- fixed_chunk = loader.parse_morphology_text(chunk_morph, name)
- fixed_chunk['description'] = fixed_chunk.pop('comments')
- loader.save_to_file(chunk['morph'], fixed_chunk)
- print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
- subprocess.call(['git', 'add', '--quiet', chunk['morph'])
- #subprocess.call(['git', 'commit', '--quiet', '-m',
- # 'Move %s into systems' % morph.filename])
- if "buildsystem" in str(err):
+ fixed_chunk = loader.parse_morphology_text(chunk_morph, name)
+ fixed_chunk['description'] = fixed_chunk.pop('comments')
+ loader.save_to_file(chunk['morph'], fixed_chunk)
+ print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
+ subprocess.call(['git', 'add', chunk['morph']])
+ #subprocess.call(['git', 'commit', '--quiet', '-m',
+ # 'Add %s to definitions repository' % name])
+ if "buildsystem" in str(err):
# This error is caused because a typo in a morphology which
# has a field "buildsystem" instead of "build-system".
- fixed_chunk = loader.parse_morphology_text(chunk_morph, name)
- fixed_chunk['build-system'] = fixed_chunk.pop('buildsystem')
- loader.save_to_file(chunk['morph'], fixed_chunk)
- print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
- subprocess.call(['git', 'add', '--quiet', chunk['morph'])
- #subprocess.call(['git', 'commit', '--quiet', '-m',
- # 'Move %s into systems' % morph.filename])
- except morphlib.morphloader.MorphologyNotYamlError as err:
- print "ERROR: %s in chunk \n%s" % (err, chunk_morph)
- # This error is caused because there are old morphologies written
- # in JSON which contain '\t' characters. When try to load this
- # kind of morphologies load_from_string fails when parse_morphology_text.
- # Removing this characters will make load_from_string to load the morphology
- # and translate it into a correct yaml format.
- fix_chunk = chunk_morph.replace('\t','')
- new_chunk = loader.load_from_string(fix_chunk)
- loader.save_to_file(chunk['morph'], new_chunk)
+ fixed_chunk = loader.parse_morphology_text(chunk_morph, name)
+ fixed_chunk['build-system'] = fixed_chunk.pop('buildsystem')
+ loader.save_to_file(chunk['morph'], fixed_chunk)
print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
- subprocess.call(['git', 'add', '--quiet', chunk['morph'])
+ subprocess.call(['git', 'add', chunk['morph']])
#subprocess.call(['git', 'commit', '--quiet', '-m',
- # 'Move %s into systems' % morph.filename])
+ # 'Add %s to definitions repository' % name])
+ except morphlib.morphloader.MorphologyNotYamlError as err:
+ print "ERROR: %s in chunk \n%s" % (err, chunk_morph)
+ # This error is caused because there are old morphologies written
+ # in JSON which contain '\t' characters. When try to load this
+ # kind of morphologies load_from_string fails when parse_morphology_text.
+ # Removing this characters will make load_from_string to load the morphology
+ # and translate it into a correct yaml format.
+ fix_chunk = chunk_morph.replace('\t','')
+ new_chunk = loader.load_from_string(fix_chunk)
+ loader.save_to_file(chunk['morph'], new_chunk)
+ print "Downloading %s from %s and placing in %s" %(name, repo, chunk['morph'])
+ subprocess.call(['git', 'add', chunk['morph']])
+ #subprocess.call(['git', 'commit', '--quiet', '-m',
+ # 'Add %s to definitions repository' % name])
+
+
+def move_strata(morphs):
+ for morph in morphs:
+ # Add chunk path on the chunk's morph field per stratum
+ stratum_path = 'strata/' + morph['name']
+ subdir = create_directory(stratum_path, definitions_repo)
+
+ # Download chunks which belongs to the stratum
+ download_chunks(morph)
# Add path to the build-depends morphologies
for build_depends in morph['build-depends']:
@@ -206,9 +204,12 @@ def move_strata(morphs):
#subprocess.call(['git', 'mv', morph.filename, new_location])
#morph.filename = new_location
#subprocess.call(['git', 'commit', '--quiet', '-m',
- # 'Move %s into subdirectory' %kind])
+ # 'Move %s into strata' % morph.filename])
+
+# Get the path from definitions repo
+definitions_repo = sb.get_git_directory_name(sb.root_repository_url)
-# Move the morphologies to it directory
+# Move the morphologies to its directories
for key in morphologies.iterkeys():
print "Moving %s....\n" %key
if key == 'cluster': move_clusters(morphologies[key])
@@ -216,6 +217,5 @@ for key in morphologies.iterkeys():
elif key == 'stratum': move_strata(morphologies[key])
elif key == 'chunk': move_chunks(morphologies[key])
else: print 'ERROR: Morphology not know %' % key
- #move_morphs(morphologies[key], key)
sys.exit(0)