# Copyright (C) 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . '''parse.py Load a set of Baserock definitions from on-disk .morph files, and return an RDFLib.Graph instance containing the data. This code understands the syntax of Baserock Definitions format version 5. The current version of the Baserock Definitions format is defined at: http://wiki.baserock.org/definitions/current ''' import rdflib import rdflib.collection import yaml import os import warnings BASEROCK = rdflib.Namespace('http://baserock.org/definitions/example-schema#') DUBLIN_CORE = rdflib.Namespace('http://purl.org/dc/terms/') def parse_morph_file(path): '''Parse an individual .morph file. This function does a tiny amount of validation: checking the 'name' and 'type' fields. Returns a Python dict with the entire contents of the file deserialised from YAML. ''' with open(path) as f: text = f.read() contents = yaml.safe_load(text) assert 'name' in contents assert contents['kind'] in ['cluster', 'system', 'stratum', 'chunk'] return contents def get_name_from_morph_file(path): '''Returns the 'name' defined in a specific .morph file. This is a convenience function for resolving places where one .morph file in a set references another one. ''' contents = parse_morph_file(path) return contents['name'] class DefinitionsNamespace(rdflib.Namespace): '''Helpers to construct URIs for Baserock entities. These functions all return instances of the rdflib.URIRef() class. Example: ns = DefinitionsNamespace('http://example.com') chunk1 = ns.chunk('gcc') print(chunk1) # => rdflib.URIRef('http://example.com/chunks/gcc') ''' def chunk(self, chunk_name): return self.term('chunks/' + chunk_name) def chunk_reference(self, stratum_uriref, chunk_name): assert isinstance(stratum_uriref, rdflib.URIRef) return stratum_uriref + '/chunk-refs/' + chunk_name def cluster(self, cluster_name): return self.term('clusters/' + cluster_name) def stratum(self, stratum_name): return self.term('strata/' + stratum_name) def stratum_artifact(self, stratum_uriref, artifact_name): assert isinstance(stratum_uriref, rdflib.URIRef) return stratum_uriref + '/products/' + artifact_name def system(self, system_name): return self.term('systems/' + system_name) def system_deployment(self, cluster_uriref, label): assert isinstance(cluster_uriref, rdflib.URIRef) return cluster_uriref + '/' + label def ordered(graph, value_list, node=None): '''Create an ordered RDF collection from a list of values. To define an ordered list in RDF, you need an identifier for the list itself. By default this function will create a 'blank node', which will end up with a random unique URI when serialised. To make serialised data more readable you can create your own rdflib.URIRef term to identify the list. ''' node = node or rdflib.BNode() collection = rdflib.collection.Collection( graph, node, [rdflib.Literal(v) for v in value_list]) return node def load_all_morphologies(path='.'): '''Load Baserock Definitions serialisation format V5 as an RDFLib 'graph'. This code does very little validation, so the 'graph' that it returns may not fully make sense according to the Baserock data model. ''' toplevel_path = path graph = rdflib.Graph() def load_morph(toplevel_path, filename): try: contents = parse_morph_file(filename) except Exception as e: warnings.warn("Problem loading %s: %s" % (filename, e)) # FIXME: caller should pass in the namespace, or we should make one # up based on os.path.abspath(path) ns = DefinitionsNamespace('http://example.com/') entity = None if contents['kind'] == 'chunk': chunk_uriref = ns.chunk(contents['name']) entity = chunk = rdflib.resource.Resource(graph, chunk_uriref) if 'pre-configure-commands' in contents: chunk.set(BASEROCK.preConfigureCommands, ordered(graph, contents['pre-configure-commands'])) if 'configure-commands' in contents: chunk.set(BASEROCK.configureCommands, ordered(graph, contents['configure-commands'])) if 'post-configure-commands' in contents: chunk.set(BASEROCK.postConfigureCommands, ordered(graph, contents['post-configure-commands'])) if 'pre-build-commands' in contents: chunk.set(BASEROCK.preBuildCommands, ordered(graph, contents['pre-build-commands'])) if 'build-commands' in contents: chunk.set(BASEROCK.buildCommands, ordered(graph, contents['build-commands'])) if 'post-build-commands' in contents: chunk.set(BASEROCK.postBuildCommands, ordered(graph, contents['post-build-commands'])) if 'pre-install-commands' in contents: chunk.set(BASEROCK.preInstallCommands, ordered(graph, contents['pre-install-commands'])) if 'install-commands' in contents: chunk.set(BASEROCK.installCommands, ordered(graph, contents['install-commands'])) if 'post-install-commands' in contents: chunk.set(BASEROCK.postInstallCommands, ordered(graph, contents['post-install-commands'])) elif contents['kind'] == 'stratum': stratum_uriref = ns.stratum(contents['name']) entity = stratum = rdflib.resource.Resource(graph, stratum_uriref) for entry in contents.get('build-depends', []): build_dep_file = os.path.join(toplevel_path, entry['morph']) build_dep_name = get_name_from_morph_file(build_dep_file) build_dep_uriref = ns.stratum(build_dep_name) stratum.add(BASEROCK.hasBuildDependency, build_dep_uriref) for entry in contents.get('products', []): artifact_uri = ns.stratum_artifact( stratum_uriref, entry['artifact']) artifact = rdflib.resource.Resource(graph, artifact_uri) # FIXME: order probably lost here if 'includes' in entry: artifact.set(BASEROCK.includes, rdflib.Literal(entry['includes'])) stratum.add(BASEROCK.produces, artifact) for entry in contents.get('chunks', []): if 'morph' in entry: chunk_file = os.path.join(toplevel_path, entry['morph']) chunk_name = get_name_from_morph_file(chunk_file) if chunk_name != entry['name']: warnings.warn( "Chunk name %s in stratum %s doesn't match " "name from %s" % (entry['name'], stratum_uriref, entry['morph'])) else: chunk_name = entry['name'] chunk_ref_uriref = ns.chunk_reference( stratum_uriref, chunk_name) chunk_ref = rdflib.resource.Resource(graph, chunk_ref_uriref) chunk_uriref = ns.chunk(chunk_name) chunk_ref.set(BASEROCK.refersToChunk, chunk_uriref) chunk_ref.set(BASEROCK.repo, rdflib.Literal(entry['repo'])) chunk_ref.set(BASEROCK.ref, rdflib.Literal(entry['ref'])) if 'unpetrify-ref' in entry: chunk_ref.set(BASEROCK.unpetrifyRef, rdflib.Literal(entry['unpetrify-ref'])) chunk_ref.set(BASEROCK.buildMode, rdflib.Literal(entry.get('build-mode', 'normal'))) chunk_ref.set(BASEROCK.prefix, rdflib.Literal(entry.get('prefix', '/usr'))) for entry_dep in entry.get('build-depends', []): build_dep_uriref = ns.chunk_reference( stratum_uriref, entry_dep) chunk_ref.set(BASEROCK.hasChunkBuildDependency, build_dep_uriref) stratum.add(BASEROCK.containsChunkReference, chunk_ref) elif contents['kind'] == 'system': system_uriref = ns.system(contents['name']) entity = system = rdflib.resource.Resource(graph, system_uriref) system.set(BASEROCK.arch, rdflib.Literal(contents['arch'])) for entry in contents.get('strata', []): # FIXME: need to include all strata if 'artifacts' isn't specified, # which is difficult becausee they might not all be loaded yet ... # so for now I cheat and just assume -runtime and -devel. If there # are extra artifacts for the stratum they won't be incuded by # default. I'm not sure if this is how Morph behaves or not. artifacts = entry.get('artifacts') if artifacts is None: artifacts = ['%s-runtime' % entry['name'], '%s-devel' % entry['name']] for artifact in artifacts: artifact_uriref = ns.stratum_artifact( ns.stratum(entry['name']), artifact) system.add(BASEROCK.containsStratumArtifact, artifact_uriref) system.set(BASEROCK.hasConfigurationExtension, rdflib.Literal( contents.get('configuration-extensions', []))) elif contents['kind'] == 'cluster': cluster_uriref = ns.cluster(contents['name']) entity = cluster = rdflib.resource.Resource(graph, cluster_uriref) for entry in contents.get('systems', []): system_morph_file = os.path.join(toplevel_path, entry['morph']) system_name = get_name_from_morph_file(system_morph_file) system_uriref = ns.system(system_name) # FIXME: ignores deploy-defaults at present for label, details in entry['deploy'].items(): deployment_uriref = ns.system_deployment( cluster_uriref, label) deployment = rdflib.resource.Resource( graph, deployment_uriref) deployment.set(BASEROCK.deploysSystem, system_uriref) deployment.set(BASEROCK.hasLabel, rdflib.Literal(label)) deployment.set(BASEROCK.hasType, rdflib.Literal(details['type'])) deployment.set(BASEROCK.hasLocation, rdflib.Literal(details['location'])) settings = [] for key, value in details.items(): if key in ['type', 'location']: continue # FIXME: RDF must have a way of representing arbitrary # key/values better than using a string with an = # sign... settings.append('%s=%s' % (key, value)) deployment.set(BASEROCK.hasConfigurationSetting, rdflib.Literal(settings)) cluster.add(BASEROCK.deploysSystem, deployment) if 'description' in contents: entity.set(DUBLIN_CORE.description, rdflib.Literal(contents['description'])) # FIXME: comments from the .yaml file are lost ... as a quick solution, # you could manually find every line from the YAML that starts with a # '#' and dump that into a property. Or ruamel.yaml might help? print 'Parsing .morph files...' for dirname, dirnames, filenames in os.walk(toplevel_path): if '.git' in dirnames: dirnames.remove('.git') for filename in sorted(filenames): if filename.endswith('.morph'): try: load_morph(toplevel_path, os.path.join(dirname, filename)) except Exception as e: print '%s: %r' % (filename, e) raise return graph