path: root/schema/parse.py

# Copyright (C) 2015  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.


'''parse.py

Load a set of Baserock definitions from on-disk .morph files, and return an
RDFLib.Graph instance containing the data.

This code understands the syntax of Baserock Definitions format version 5.

The current version of the Baserock Definitions format is defined at:

    http://wiki.baserock.org/definitions/current

'''


import rdflib
import yaml

import os
import warnings


BASEROCK = rdflib.Namespace('http://baserock.org/definitions/example-schema#')
DUBLIN_CORE = rdflib.Namespace('http://purl.org/dc/terms/')


def parse_morph_file(path):
    '''Parse an individual .morph file.

    This function does a tiny amount of validation: checking the 'name' and
    'type' fields.

    Returns a Python dict with the entire contents of the file deserialised
    from YAML.

    '''
    with open(path) as f:
        text = f.read()
    contents = yaml.safe_load(text)
    assert 'name' in contents
    assert contents['kind'] in ['cluster', 'system', 'stratum', 'chunk']
    return contents


def get_name_from_morph_file(path):
    '''Returns the 'name' defined in a specific .morph file.

    This is a convenience function for resolving places where one .morph file
    in a set references another one.

    '''
    contents = parse_morph_file(path)
    return contents['name']


# FIXME: get_uri_for_resource()


# FIXME: can you use assignment instead of Resource.set ?

def load_all_morphologies(path='.'):
    '''Load Baserock Definitions serialisation format V5 as an RDFLib 'graph'.

    This code does very little validation, so the 'graph' that it returns may
    not fully make sense according to the Baserock data model.

    '''
    toplevel_path = path
    graph = rdflib.Graph()

    def load_morph(toplevel_path, filename):
        try:
            contents = parse_morph_file(filename)
        except Exception as e:
            warnings.warn("Problem loading %s: %s" % (filename, e))

        # FIXME:
        base_uri = 'http://example.com/'

        entity = None

        if contents['kind'] == 'chunk':
            chunk_uri = base_uri + 'chunks/' + contents['name']
            entity = chunk = rdflib.resource.Resource(
                graph, rdflib.URIRef(chunk_uri))

            # FIXME: order is lost here !!!!!
            if 'pre-configure-commands' in contents:
                chunk.add(BASEROCK.preConfigureCommands,
                          rdflib.Literal(contents['pre-configure-commands']))
            if 'configure-commands' in contents:
                chunk.add(BASEROCK.configureCommands,
                          rdflib.Literal(contents['configure-commands']))
            if 'post-configure-commands' in contents:
                chunk.add(BASEROCK.postConfigureCommands,
                          rdflib.Literal(contents['post-configure-commands']))
            if 'pre-build-commands' in contents:
                chunk.add(BASEROCK.preBuildCommands,
                          rdflib.Literal(contents['pre-build-commands']))
            if 'build-commands' in contents:
                chunk.add(BASEROCK.buildCommands,
                          rdflib.Literal(contents['build-commands']))
            if 'post-build-commands' in contents:
                chunk.add(BASEROCK.postBuildCommands,
                          rdflib.Literal(contents['post-build-commands']))
            if 'pre-install-commands' in contents:
                chunk.add(BASEROCK.preInstallCommands,
                          rdflib.Literal(contents['pre-install-commands']))
            if 'install-commands' in contents:
                chunk.add(BASEROCK.installCommands,
                          rdflib.Literal(contents['install-commands']))
            if 'post-install-commands' in contents:
                chunk.add(BASEROCK.postInstallCommands,
                          rdflib.Literal(contents['post-install-commands']))

        elif contents['kind'] == 'stratum':
            stratum_uri = base_uri + 'strata/' + contents['name']
            entity = stratum = rdflib.resource.Resource(
                graph, rdflib.URIRef(stratum_uri))

            for entry in contents.get('build-depends', []):
                build_dep_file = os.path.join(toplevel_path, entry['morph'])
                build_dep_name = get_name_from_morph_file(build_dep_file)
                build_dep_uri = base_uri + 'strata/' + build_dep_name
                stratum.add(BASEROCK.hasBuildDependency,
                            rdflib.URIRef(build_dep_uri))

            for entry in contents.get('products', []):
                artifact_uri = stratum_uri + '/products/' + entry['artifact']
                artifact = rdflib.resource.Resource(
                    graph, rdflib.URIRef(artifact_uri))
                # FIXME: order probably lost here
                if 'includes' in entry:
                    artifact.set(BASEROCK.includes,
                                 rdflib.Literal(entry['includes']))
                stratum.add(BASEROCK.produces, artifact)

            chunk_refs = []
            for entry in contents.get('chunks', []):
                chunk_ref_uri = stratum_uri + '/chunk-refs/' + entry['name']
                chunk_ref = rdflib.resource.Resource(
                    graph, rdflib.URIRef(chunk_ref_uri))

                if 'morph' in entry:
                    chunk_file = os.path.join(toplevel_path, entry['morph'])
                    chunk_name = get_name_from_morph_file(chunk_file)
                    if chunk_name != entry['name']:
                        warnings.warn(
                            "Chunk name %s in stratum %s doesn't match "
                            "name from %s" % (entry['name'], stratum_uri,
                                              entry['morph']))
                else:
                    chunk_name = entry['name']

                chunk_uri = base_uri + 'chunks/' + chunk_name
                chunk_ref.set(BASEROCK.refersToChunk, rdflib.URIRef(chunk_uri))

                chunk_ref.set(BASEROCK.repo, rdflib.Literal(entry['repo']))
                chunk_ref.set(BASEROCK.ref, rdflib.Literal(entry['ref']))
                if 'unpetrify-ref' in entry:
                    chunk_ref.set(BASEROCK.unpetrifyRef,
                              rdflib.Literal(entry['unpetrify-ref']))
                chunk_ref.set(BASEROCK.buildMode,
                          rdflib.Literal(entry.get('build-mode', 'normal')))
                chunk_ref.set(BASEROCK.prefix,
                          rdflib.Literal(entry.get('prefix', '/usr')))

                for entry_dep in entry.get('build-depends', []):
                    build_dep_uri = stratum_uri + '/chunk-refs/' + entry_dep
                    chunk_ref.set(BASEROCK.hasChunkBuildDependency,
                                  rdflib.URIRef(build_dep_uri))

                stratum.add(BASEROCK.containsChunkReference, chunk_ref)

        elif contents['kind'] == 'system':
            system_uri = base_uri + 'systems/' + contents['name']
            entity = system = rdflib.resource.Resource(
                graph, rdflib.URIRef(system_uri))

            system.set(BASEROCK.arch, rdflib.Literal(contents['arch']))

            for entry in contents.get('strata', []):
                # FIXME: need to include all strata if 'artifacts' isn't specified,
                # which is difficult becausee they might not all be loaded yet ...
                # so for now I cheat and just assume -runtime and -devel. If there
                # are extra artifacts for the stratum they won't be incuded by
                # default. I'm not sure if this is how Morph behaves or not.
                artifacts = entry.get('artifacts')
                if artifacts is None:
                    artifacts = ['%s-runtime' % entry['name'],
                                 '%s-devel' % entry['name']]
                for artifact in artifacts:
                    artifact_uri = (base_uri + '/strata/' + entry['name'] +
                                    '/products/' + artifact)
                    system.add(BASEROCK.containsStratumArtifact,
                               rdflib.URIRef(artifact_uri))

            system.set(BASEROCK.hasConfigurationExtension, rdflib.Literal(
                contents.get('configuration-extensions', [])))

        elif contents['kind'] == 'cluster':
            cluster_uri = base_uri + 'clusters/' + contents['name']
            entity = cluster = rdflib.resource.Resource(
                graph, rdflib.URIRef(cluster_uri))

            deployments = []
            for entry in contents.get('systems', []):
                system_morph_file = os.path.join(toplevel_path, entry['morph'])
                system_name = get_name_from_morph_file(system_morph_file)
                system_uri = base_uri + 'systems/' + system_name

                # FIXME: ignores deploy-defaults at present
                for label, details in entry['deploy'].items():
                    deployment_uri = cluster_uri + '/' + label
                    deployment = rdflib.resource.Resource(
                        graph, rdflib.URIRef(deployment_uri))

                    deployment.set(BASEROCK.deploysSystem,
                                   rdflib.URIRef(system_uri))
                    deployment.set(BASEROCK.hasLabel, rdflib.Literal(label))

                    deployment.set(BASEROCK.hasType,
                                   rdflib.Literal(details['type']))
                    deployment.set(BASEROCK.hasLocation,
                                   rdflib.Literal(details['location']))

                    settings = []
                    for key, value in details.items():
                        if key in ['type', 'location']:
                            continue
                        # FIXME: RDF must have a way of representing arbitrary
                        # key/values better than using a string with an =
                        # sign...
                        settings.append('%s=%s' % (key, value))
                    deployment.set(BASEROCK.hasConfigurationSetting,
                                   rdflib.Literal(settings))

                    cluster.add(BASEROCK.deploysSystem, deployment)

        if 'description' in contents:
            entity.set(DUBLIN_CORE.description,
                       rdflib.Literal(contents['description']))

        # FIXME: comments from the .yaml file are lost ... as a quick solution,
        # you could manually find every line from the YAML that starts with a
        # '#' and dump that into a property. Or ruamel.yaml might help?

    print 'Parsing .morph files...'
    for dirname, dirnames, filenames in os.walk(toplevel_path):
        if '.git' in dirnames:
            dirnames.remove('.git')
        for filename in sorted(filenames):
            if filename.endswith('.morph'):
                try:
                    load_morph(toplevel_path, os.path.join(dirname, filename))
                except Exception as e:
                    print '%s: %r' % (filename, e)
                    raise

    return graph