# Copyright (C) 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . '''parse.py Load a set of Baserock definitions from on-disk .morph files, and return an RDFLib.Graph instance containing the data. This code understands the syntax of Baserock Definitions format version 5. The current version of the Baserock Definitions format is defined at: http://wiki.baserock.org/definitions/current ''' import rdflib import yaml import os import warnings BASEROCK = rdflib.Namespace('http://baserock.org/definitions/example-schema#') DUBLIN_CORE = rdflib.Namespace('http://purl.org/dc/terms/') def parse_morph_file(path): '''Parse an individual .morph file. This function does a tiny amount of validation: checking the 'name' and 'type' fields. Returns a Python dict with the entire contents of the file deserialised from YAML. ''' with open(path) as f: text = f.read() contents = yaml.safe_load(text) assert 'name' in contents assert contents['kind'] in ['cluster', 'system', 'stratum', 'chunk'] return contents def get_name_from_morph_file(path): '''Returns the 'name' defined in a specific .morph file. This is a convenience function for resolving places where one .morph file in a set references another one. ''' contents = parse_morph_file(path) return contents['name'] # FIXME: can you use assignment instead of Resource.set ? def uriref_for_chunk(base_uri, chunk_name): uriref = base_uri + 'chunks/' + chunk_name return rdflib.URIRef(uriref) def uriref_for_chunk_reference(base_uri, stratum_name, chunk_name): stratum_uriref = uriref_for_stratum(base_uri, stratum_name) chunk_reference_uriref = stratum_uriref + '/chunk-refs/' + chunk_name return rdflib.URIRef(chunk_reference_uriref) def uriref_for_cluster(base_uri, cluster_name): uriref = base_uri + 'clusters/' + cluster_name return rdflib.URIRef(uriref) def uriref_for_stratum(base_uri, stratum_name): uriref = base_uri + 'strata/' + stratum_name return rdflib.URIRef(uriref) def uriref_for_stratum_artifact(base_uri, stratum_name, artifact_name): stratum_uriref = uriref_for_stratum(base_uri, stratum_name) artifact_uriref = stratum_uriref + '/products/' + artifact_name return rdflib.URIRef(artifact_uriref) def uriref_for_system(base_uri, system_name): uriref = base_uri + 'systems/' + system_name return rdflib.URIRef(uriref) def uriref_for_system_deployment(base_uri, cluster_name, label): cluster_uriref = uriref_for_cluster(base_uri, cluster_name) system_deployment_uriref = cluster_uriref + label return rdflib.URIRef(system_deployment_uriref) def load_all_morphologies(path='.'): '''Load Baserock Definitions serialisation format V5 as an RDFLib 'graph'. This code does very little validation, so the 'graph' that it returns may not fully make sense according to the Baserock data model. ''' toplevel_path = path graph = rdflib.Graph() def load_morph(toplevel_path, filename): try: contents = parse_morph_file(filename) except Exception as e: warnings.warn("Problem loading %s: %s" % (filename, e)) # FIXME: base_uri = 'http://example.com/' entity = None if contents['kind'] == 'chunk': chunk_uriref = uriref_for_chunk(base_uri, contents['name']) entity = chunk = rdflib.resource.Resource(graph, chunk_uriref) # FIXME: order is lost here !!!!! if 'pre-configure-commands' in contents: chunk.add(BASEROCK.preConfigureCommands, rdflib.Literal(contents['pre-configure-commands'])) if 'configure-commands' in contents: chunk.add(BASEROCK.configureCommands, rdflib.Literal(contents['configure-commands'])) if 'post-configure-commands' in contents: chunk.add(BASEROCK.postConfigureCommands, rdflib.Literal(contents['post-configure-commands'])) if 'pre-build-commands' in contents: chunk.add(BASEROCK.preBuildCommands, rdflib.Literal(contents['pre-build-commands'])) if 'build-commands' in contents: chunk.add(BASEROCK.buildCommands, rdflib.Literal(contents['build-commands'])) if 'post-build-commands' in contents: chunk.add(BASEROCK.postBuildCommands, rdflib.Literal(contents['post-build-commands'])) if 'pre-install-commands' in contents: chunk.add(BASEROCK.preInstallCommands, rdflib.Literal(contents['pre-install-commands'])) if 'install-commands' in contents: chunk.add(BASEROCK.installCommands, rdflib.Literal(contents['install-commands'])) if 'post-install-commands' in contents: chunk.add(BASEROCK.postInstallCommands, rdflib.Literal(contents['post-install-commands'])) elif contents['kind'] == 'stratum': stratum_name = contents['name'] stratum_uriref = uriref_for_stratum(base_uri, stratum_name) entity = stratum = rdflib.resource.Resource(graph, stratum_uriref) for entry in contents.get('build-depends', []): build_dep_file = os.path.join(toplevel_path, entry['morph']) build_dep_name = get_name_from_morph_file(build_dep_file) build_dep_uriref = uriref_for_stratum(base_uri, build_dep_name) stratum.add(BASEROCK.hasBuildDependency, build_dep_uriref) for entry in contents.get('products', []): artifact_uri = uriref_for_stratum_artifact( base_uri, stratum_name, entry['artifact']) artifact = rdflib.resource.Resource(graph, artifact_uri) # FIXME: order probably lost here if 'includes' in entry: artifact.set(BASEROCK.includes, rdflib.Literal(entry['includes'])) stratum.add(BASEROCK.produces, artifact) for entry in contents.get('chunks', []): if 'morph' in entry: chunk_file = os.path.join(toplevel_path, entry['morph']) chunk_name = get_name_from_morph_file(chunk_file) if chunk_name != entry['name']: warnings.warn( "Chunk name %s in stratum %s doesn't match " "name from %s" % (entry['name'], stratum_uriref, entry['morph'])) else: chunk_name = entry['name'] chunk_ref_uriref = uriref_for_chunk_reference( base_uri, stratum_name, chunk_name) chunk_ref = rdflib.resource.Resource(graph, chunk_ref_uriref) chunk_uriref = uriref_for_chunk(base_uri, chunk_name) chunk_ref.set(BASEROCK.refersToChunk, chunk_uriref) chunk_ref.set(BASEROCK.repo, rdflib.Literal(entry['repo'])) chunk_ref.set(BASEROCK.ref, rdflib.Literal(entry['ref'])) if 'unpetrify-ref' in entry: chunk_ref.set(BASEROCK.unpetrifyRef, rdflib.Literal(entry['unpetrify-ref'])) chunk_ref.set(BASEROCK.buildMode, rdflib.Literal(entry.get('build-mode', 'normal'))) chunk_ref.set(BASEROCK.prefix, rdflib.Literal(entry.get('prefix', '/usr'))) for entry_dep in entry.get('build-depends', []): build_dep_uriref = uriref_for_chunk_reference( base_uri, stratum_name, entry_dep) chunk_ref.set(BASEROCK.hasChunkBuildDependency, build_dep_uriref) stratum.add(BASEROCK.containsChunkReference, chunk_ref) elif contents['kind'] == 'system': system_uriref = uriref_for_system(base_uri, contents['name']) entity = system = rdflib.resource.Resource(graph, system_uriref) system.set(BASEROCK.arch, rdflib.Literal(contents['arch'])) for entry in contents.get('strata', []): # FIXME: need to include all strata if 'artifacts' isn't specified, # which is difficult becausee they might not all be loaded yet ... # so for now I cheat and just assume -runtime and -devel. If there # are extra artifacts for the stratum they won't be incuded by # default. I'm not sure if this is how Morph behaves or not. artifacts = entry.get('artifacts') if artifacts is None: artifacts = ['%s-runtime' % entry['name'], '%s-devel' % entry['name']] for artifact in artifacts: artifact_uriref = uriref_for_stratum_artifact( base_uri, entry['name'], artifact) system.add(BASEROCK.containsStratumArtifact, artifact_uriref) system.set(BASEROCK.hasConfigurationExtension, rdflib.Literal( contents.get('configuration-extensions', []))) elif contents['kind'] == 'cluster': cluster_name = contents['name'] cluster_uriref = uriref_for_cluster(base_uri, cluster_name) entity = cluster = rdflib.resource.Resource(graph, cluster_uriref) for entry in contents.get('systems', []): system_morph_file = os.path.join(toplevel_path, entry['morph']) system_name = get_name_from_morph_file(system_morph_file) system_uriref = uriref_for_system(base_uri, system_name) # FIXME: ignores deploy-defaults at present for label, details in entry['deploy'].items(): deployment_uriref = uriref_for_system_deployment( base_uri, cluster_name, label) deployment = rdflib.resource.Resource( graph, deployment_uriref) deployment.set(BASEROCK.deploysSystem, system_uriref) deployment.set(BASEROCK.hasLabel, rdflib.Literal(label)) deployment.set(BASEROCK.hasType, rdflib.Literal(details['type'])) deployment.set(BASEROCK.hasLocation, rdflib.Literal(details['location'])) settings = [] for key, value in details.items(): if key in ['type', 'location']: continue # FIXME: RDF must have a way of representing arbitrary # key/values better than using a string with an = # sign... settings.append('%s=%s' % (key, value)) deployment.set(BASEROCK.hasConfigurationSetting, rdflib.Literal(settings)) cluster.add(BASEROCK.deploysSystem, deployment) if 'description' in contents: entity.set(DUBLIN_CORE.description, rdflib.Literal(contents['description'])) # FIXME: comments from the .yaml file are lost ... as a quick solution, # you could manually find every line from the YAML that starts with a # '#' and dump that into a property. Or ruamel.yaml might help? print 'Parsing .morph files...' for dirname, dirnames, filenames in os.walk(toplevel_path): if '.git' in dirnames: dirnames.remove('.git') for filename in sorted(filenames): if filename.endswith('.morph'): try: load_morph(toplevel_path, os.path.join(dirname, filename)) except Exception as e: print '%s: %r' % (filename, e) raise return graph