Merge branch 'lc/rework_splitting' into lc/staging/future

2 files changed, 71 insertions, 232 deletions

diff --git a/ybd/assembly.py b/ybd/assembly.py
index 34f278a..d910bdb 100644
--- a/ybd/assembly.py
+++ b/ybd/assembly.py
@@ -27,7 +27,9 @@ from cache import cache, cache_key, get_cache, get_remote
 import repos
 import sandbox
 import datetime
-from splitting import write_metadata, install_split_artifacts
+import shutil
+import yaml
+import utils
 
 
 def compose(dn):
@@ -134,6 +136,68 @@ def install_dependencies(dn, dependencies=None):
         sandbox.list_files(dn)
 
 
+def path_to_metafile(dn):
+    ''' Return the path to metadata file for dn. '''
+
+    return os.path.join(get_cache(dn) + '.unpacked', 'baserock',
+                        dn['name'] + '.meta')
+
+
+def write_chunk_metafile(chunk):
+    '''Writes a chunk .meta file to the baserock dir of the chunk.'''
+    log(chunk['name'], 'Writing Chunk Metafile')
+
+    metadata = {'repo': chunk.get('repo'),
+                'ref': chunk.get('ref'),
+                'cache': chunk.get('cache')}
+
+    meta = os.path.join(chunk['baserockdir'], chunk['name'] + '.meta')
+
+    with open(meta, "w") as f:
+        yaml.safe_dump(metadata, f, default_flow_style=False)
+
+
+def write_stratum_metafiles(stratum):
+    '''Write the .meta files for a stratum to the baserock dir.'''
+    log(stratum['name'], 'Writing Stratum Metafile')
+
+    for x in stratum['contents']:
+        d = app.defs.get(x)
+        if d.get('build-mode', 'staging') == 'bootstrap':
+            continue
+        p = path_to_metafile(d)
+        f = os.path.join(stratum['baserockdir'], d['name'] + '.meta')
+        shutil.copyfile(p, f)
+
+    metadata = {'repo': config['defdir'],
+                'ref': config['def-version'],
+                'cache': stratum['cache']}
+
+    meta = os.path.join(stratum['baserockdir'], stratum['name'] + 'meta')
+    with open(meta, "w") as f:
+        yaml.safe_dump(metadata, f, default_flow_style=False)
+
+
+def get_includes_iterator(dn):
+    f = [app.defs.get(x)
+         for x in dn['contents']
+         if app.defs.get(x).get('build-mode', 'staging') == 'staging']
+    for y in f:
+        if y.get('kind', 'chunk') == 'chunk':
+            yield y
+        else:
+            g = list(get_includes_iterator(y))
+            for z in g:
+                yield z
+
+
+def install_all_artifacts(dn):
+    for x in get_includes_iterator(dn):
+        a = get_cache(x) + '.unpacked'
+        meta = os.path.join(dn['baserockdir'], x['name'] + '.meta')
+        utils.copy_all_files(a, dn['install'])
+
+
 def build(dn):
     '''Create an artifact for a single component and add it to the cache'''
 
@@ -148,10 +212,13 @@ def build(dn):
 
         with timer(dn, 'artifact creation'):
 
+            if dn.get('kind', 'chunk') == 'chunk':
+                write_chunk_metafile(dn)
+            if dn.get('kind', 'stratum') == 'stratum':
+                write_stratum_metafiles(dn)
             if dn.get('kind', 'chunk') == 'system':
-                install_split_artifacts(dn)
+                install_all_artifacts(dn)
 
-            write_metadata(dn)
             cache(dn)
 
 
diff --git a/ybd/splitting.py b/ybd/splitting.py
index 7c5f8d1..3f4b7c0 100644
--- a/ybd/splitting.py
+++ b/ybd/splitting.py
@@ -22,142 +22,9 @@ import re
 import yaml
 from utils import copy_file_list
 from fs.osfs import OSFS
-import itertools
-
-
-def install_split_artifacts(dn):
-    '''Create the .meta files for a split system
-
-    Given a list of artifacts to split, writes new .meta files to
-    the baserock dir in dn['install'] and copies the files from the
-    sandbox to the dn['install']
-
-    '''
-
-    for content in dn['contents']:
-        key = content.keys()[0]
-        stratum = app.defs.get(key)
-        move_required_files(dn, stratum, content[key])
-
-
-def move_required_files(dn, stratum, artifacts):
-    stratum_metadata = get_metadata(stratum)
-    split_stratum_metadata = {}
-    if not artifacts:
-        # Include all artifacts if no ones were explicitly given for an
-        # included stratum on a system.
-        artifacts = [p['artifact'] for p in stratum_metadata['products']]
-
-    to_keep = [component
-               for product in stratum_metadata['products']
-               for component in product['components']
-               if product['artifact'] in artifacts]
-
-    split_stratum_metadata['products'] = (
-              [product
-               for product in stratum_metadata['products']
-               if product['artifact'] in artifacts])
-
-    log(dn, 'Installing %s artifacts' % stratum['name'], artifacts)
-    log(dn, 'Installing components:', to_keep, verbose=True)
-
-    baserockpath = os.path.join(dn['install'], 'baserock')
-    if not os.path.isdir(baserockpath):
-        os.mkdir(baserockpath)
-    split_stratum_metafile = os.path.join(baserockpath,
-                                          stratum['name'] + '.meta')
-    with open(split_stratum_metafile, "w") as f:
-        yaml.safe_dump(split_stratum_metadata, f, default_flow_style=False)
-
-    for path in stratum['contents']:
-        chunk = app.defs.get(path)
-        if chunk.get('build-mode', 'staging') == 'bootstrap':
-            continue
-
-        try:
-            metafile = path_to_metafile(chunk)
-            with open(metafile, "r") as f:
-                filelist = []
-                metadata = yaml.safe_load(f)
-                split_metadata = {'ref': metadata.get('ref'),
-                                  'repo': metadata.get('repo'),
-                                  'products': []}
-                if config.get('artifact-version', 0) not in range(0, 1):
-                    metadata['cache'] = dn.get('cache')
-
-                for product in metadata['products']:
-                    if product['artifact'] in to_keep:
-                        filelist += product.get('components', [])
-                        # handle old artifacts still containing 'files'
-                        filelist += product.get('files', [])
-
-                        split_metadata['products'].append(product)
-
-                if split_metadata['products'] != []:
-                    split_metafile = os.path.join(baserockpath,
-                                                  os.path.basename(metafile))
-                    with open(split_metafile, "w") as f:
-                        yaml.safe_dump(split_metadata, f,
-                                       default_flow_style=False)
-                    log(dn, 'Splits split_metadata is\n', split_metadata,
-                        verbose=True)
-                    log(dn, 'Splits filelist is\n', filelist, verbose=True)
-                    copy_file_list(dn['sandbox'], dn['install'], filelist)
-        except:
-            import traceback
-            traceback.print_exc()
-            log(dn, 'Failed to install split components', exit=True)
-
-
-def check_overlaps(dn):
-    if set(config['new-overlaps']) <= set(config['overlaps']):
-        config['new-overlaps'] = []
-        return
-
-    overlaps_found = False
-    config['new-overlaps'] = list(set(config['new-overlaps']))
-    for path in config['new-overlaps']:
-        log(dn, 'WARNING: overlapping path', path)
-        for filename in os.listdir(dn['baserockdir']):
-            with open(os.path.join(dn['baserockdir'], filename)) as f:
-                for line in f:
-                    if path[1:] in line:
-                        log(filename, 'WARNING: overlap at', path[1:])
-                        overlaps_found = True
-                        break
-        if config.get('check-overlaps') == 'exit':
-            log(dn, 'Overlaps found', config['new-overlaps'], exit=True)
-    config['overlaps'] = list(set(config['new-overlaps'] + config['overlaps']))
-    config['new-overlaps'] = []
-
-
-def get_metadata(dn):
-    '''Load an individual .meta file
-
-    The .meta file is expected to be in the .unpacked/baserock directory of the
-    built artifact
-
-    '''
-    try:
-        with open(path_to_metafile(dn), "r") as f:
-            metadata = yaml.safe_load(f)
-        log(dn, 'Loaded metadata', dn['path'], verbose=True)
-        return metadata
-    except:
-        log(dn, 'WARNING: problem loading metadata', dn)
-        return None
-
-
-def path_to_metafile(dn):
-    ''' Return the path to metadata file for dn. '''
-
-    return os.path.join(get_cache(dn) + '.unpacked', 'baserock',
-                        dn['name'] + '.meta')
 
 
 def compile_rules(idn, rules):
-    '''Takes an identifying string and a set of yaml splitting rules and
-       compiles them into list of (artifact, regex) pairs.'''
     regexps = []
     for rule in rules:
         regexp = re.compile('^(?:' + '|'.join(rule.get('include')) + ')$')
@@ -169,106 +36,11 @@ def compile_rules(idn, rules):
 
 
 def split_on_folder(rules, folder):
-    '''Takes a set of compiled rules and a folder name and returns a
-       dictionary of {artifact:[path]}.'''
     f = OSFS(folder)
     splits = dict((a[0], []) for a in rules)
-    for path in itertools.chain(f.walkfiles(), f.walkdirs()):
+    for path in f.walkfiles(), f.walkdirs():
         for artifact, rule in rules:
             if rule.match(path) or rule.match(path + '/'):
                 splits[artifact].append(path)
                 break
     return splits
-
-
-def write_metadata(dn):
-    if dn.get('kind', 'chunk') == 'chunk':
-        write_chunk_metafile(dn)
-    elif dn.get('kind', 'chunk') == 'stratum':
-        write_stratum_metafiles(dn)
-    if config.get('check-overlaps', 'ignore') != 'ignore':
-        check_overlaps(dn)
-
-
-def write_chunk_metafile(chunk):
-    '''Writes a chunk .meta file to the baserock dir of the chunk
-
-    The split rules are used to divide up the installed files for the chunk
-    into artifacts in the 'products' list
-
-    '''
-    log(chunk['name'], 'Splitting Chunk')
-    rules = (compile_rules(chunk['name'], chunk.get('products', [])) +
-             compile_rules(chunk['name'],
-                           app.defs.defaults.get_split_rules('chunk')))
-    splits = split_on_folder(rules, chunk['install'])
-    write_metafile(rules, splits, chunk)
-
-
-def write_stratum_metafiles(stratum):
-    '''Write the .meta files for a stratum to the baserock dir
-
-    The split rules are used to divide up the installed components into
-    artifacts in the 'products' list in the stratum .meta file. Each artifact
-    contains a list of chunk artifacts which match the stratum splitting rules
-
-    '''
-
-    log(stratum['name'], 'Splitting', stratum.get('kind'))
-    rules = (compile_rules(stratum['name'], stratum.get('products', [])) +
-             compile_rules(stratum['name'],
-                           app.defs.defaults.get_split_rules('stratum')))
-    splits = dict((a[0], []) for a in rules)
-
-    for item in stratum['contents']:
-        chunk = app.defs.get(item)
-        if chunk.get('build-mode', 'staging') == 'bootstrap':
-            continue
-
-        metadata = get_metadata(chunk)
-        split_metadata = {'ref': metadata.get('ref'),
-                          'repo': metadata.get('repo'),
-                          'products': []}
-
-        if config.get('artifact-version', 0) not in range(0, 1):
-            split_metadata['cache'] = metadata.get('cache')
-
-        chunk_artifacts = app.defs.get(chunk).get('artifacts', {})
-        for artifact, target in chunk_artifacts.items():
-            splits[target].append(artifact)
-
-        for product in metadata['products']:
-            for artifact, rule in rules:
-                if rule.match(product['artifact']):
-                    split_metadata['products'].append(product)
-                    splits[artifact].append(product['artifact'])
-                    break
-
-        meta = os.path.join(stratum['baserockdir'], chunk['name'] + '.meta')
-
-        with open(meta, "w") as f:
-            yaml.safe_dump(split_metadata, f, default_flow_style=False)
-
-    write_metafile(rules, splits, stratum)
-
-
-def write_metafile(rules, splits, dn):
-    metadata = {'products': [{'artifact': a,
-                              'components': sorted(set(splits[a]))}
-                             for a, r in rules]}
-
-    if dn.get('kind', 'chunk') == 'chunk':
-        metadata['repo'] = dn.get('repo')
-        metadata['ref'] = dn.get('ref')
-    else:
-        if config.get('artifact-version', 0) not in range(0, 2):
-            metadata['repo'] = config['defdir']
-            metadata['ref'] = config['def-version']
-
-    if config.get('artifact-version', 0) not in range(0, 1):
-        metadata['cache'] = dn.get('cache')
-
-    meta = os.path.join(dn['baserockdir'], dn['name'] + '.meta')
-
-    with open(meta, "w") as f:
-        yaml.safe_dump(metadata, f, default_flow_style=False)