# Copyright (C) 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . '''parse.py Load a set of Baserock definitions from on-disk .morph files, and return an RDFLib.Graph instance containing the data. This code understands the syntax of Baserock Definitions format version 5. The current version of the Baserock Definitions format is defined at: http://wiki.baserock.org/definitions/current ''' import rdflib import yaml import os import warnings BASEROCK = rdflib.Namespace('http://baserock.org/definitions/example-schema#') DUBLIN_CORE = rdflib.Namespace('http://purl.org/dc/terms/') def parse_morph_file(path): '''Parse an individual .morph file. This function does a tiny amount of validation: checking the 'name' and 'type' fields. Returns a Python dict with the entire contents of the file deserialised from YAML. ''' with open(path) as f: text = f.read() contents = yaml.safe_load(text) assert 'name' in contents assert contents['kind'] in ['cluster', 'system', 'stratum', 'chunk'] return contents def get_name_from_morph_file(path): '''Returns the 'name' defined in a specific .morph file. This is a convenience function for resolving places where one .morph file in a set references another one. ''' contents = parse_morph_file(path) return contents['name'] # FIXME: get_uri_for_resource() # FIXME: can you use assignment instead of Resource.set ? def load_all_morphologies(path='.'): '''Load Baserock Definitions serialisation format V5 as an RDFLib 'graph'. This code does very little validation, so the 'graph' that it returns may not fully make sense according to the Baserock data model. ''' toplevel_path = path graph = rdflib.Graph() def load_morph(toplevel_path, filename): try: contents = parse_morph_file(filename) except Exception as e: warnings.warn("Problem loading %s: %s" % (filename, e)) # FIXME: base_uri = 'http://example.com/' entity = None if contents['kind'] == 'chunk': chunk_uri = base_uri + 'chunks/' + contents['name'] entity = chunk = rdflib.resource.Resource( graph, rdflib.URIRef(chunk_uri)) # FIXME: order is lost here !!!!! if 'pre-configure-commands' in contents: chunk.add(BASEROCK.preConfigureCommands, rdflib.Literal(contents['pre-configure-commands'])) if 'configure-commands' in contents: chunk.add(BASEROCK.configureCommands, rdflib.Literal(contents['configure-commands'])) if 'post-configure-commands' in contents: chunk.add(BASEROCK.postConfigureCommands, rdflib.Literal(contents['post-configure-commands'])) if 'pre-build-commands' in contents: chunk.add(BASEROCK.preBuildCommands, rdflib.Literal(contents['pre-build-commands'])) if 'build-commands' in contents: chunk.add(BASEROCK.buildCommands, rdflib.Literal(contents['build-commands'])) if 'post-build-commands' in contents: chunk.add(BASEROCK.postBuildCommands, rdflib.Literal(contents['post-build-commands'])) if 'pre-install-commands' in contents: chunk.add(BASEROCK.preInstallCommands, rdflib.Literal(contents['pre-install-commands'])) if 'install-commands' in contents: chunk.add(BASEROCK.installCommands, rdflib.Literal(contents['install-commands'])) if 'post-install-commands' in contents: chunk.add(BASEROCK.postInstallCommands, rdflib.Literal(contents['post-install-commands'])) elif contents['kind'] == 'stratum': stratum_uri = base_uri + 'strata/' + contents['name'] entity = stratum = rdflib.resource.Resource( graph, rdflib.URIRef(stratum_uri)) for entry in contents.get('build-depends', []): build_dep_file = os.path.join(toplevel_path, entry['morph']) build_dep_name = get_name_from_morph_file(build_dep_file) build_dep_uri = base_uri + 'strata/' + build_dep_name stratum.add(BASEROCK.hasBuildDependency, rdflib.URIRef(build_dep_uri)) for entry in contents.get('products', []): artifact_uri = stratum_uri + '/products/' + entry['artifact'] artifact = rdflib.resource.Resource( graph, rdflib.URIRef(artifact_uri)) # FIXME: order probably lost here if 'includes' in entry: artifact.set(BASEROCK.includes, rdflib.Literal(entry['includes'])) stratum.add(BASEROCK.produces, artifact) chunk_refs = [] for entry in contents.get('chunks', []): chunk_ref_uri = stratum_uri + '/chunk-refs/' + entry['name'] chunk_ref = rdflib.resource.Resource( graph, rdflib.URIRef(chunk_ref_uri)) if 'morph' in entry: chunk_file = os.path.join(toplevel_path, entry['morph']) chunk_name = get_name_from_morph_file(chunk_file) if chunk_name != entry['name']: warnings.warn( "Chunk name %s in stratum %s doesn't match " "name from %s" % (entry['name'], stratum_uri, entry['morph'])) else: chunk_name = entry['name'] chunk_uri = base_uri + 'chunks/' + chunk_name chunk_ref.set(BASEROCK.refersToChunk, rdflib.URIRef(chunk_uri)) chunk_ref.set(BASEROCK.repo, rdflib.Literal(entry['repo'])) chunk_ref.set(BASEROCK.ref, rdflib.Literal(entry['ref'])) if 'unpetrify-ref' in entry: chunk_ref.set(BASEROCK.unpetrifyRef, rdflib.Literal(entry['unpetrify-ref'])) chunk_ref.set(BASEROCK.buildMode, rdflib.Literal(entry.get('build-mode', 'normal'))) chunk_ref.set(BASEROCK.prefix, rdflib.Literal(entry.get('prefix', '/usr'))) for entry_dep in entry.get('build-depends', []): build_dep_uri = stratum_uri + '/chunk-refs/' + entry_dep chunk_ref.set(BASEROCK.hasChunkBuildDependency, rdflib.URIRef(build_dep_uri)) stratum.add(BASEROCK.containsChunkReference, chunk_ref) elif contents['kind'] == 'system': system_uri = base_uri + 'systems/' + contents['name'] entity = system = rdflib.resource.Resource( graph, rdflib.URIRef(system_uri)) system.set(BASEROCK.arch, rdflib.Literal(contents['arch'])) for entry in contents.get('strata', []): # FIXME: need to include all strata if 'artifacts' isn't specified, # which is difficult becausee they might not all be loaded yet ... # so for now I cheat and just assume -runtime and -devel. If there # are extra artifacts for the stratum they won't be incuded by # default. I'm not sure if this is how Morph behaves or not. artifacts = entry.get('artifacts') if artifacts is None: artifacts = ['%s-runtime' % entry['name'], '%s-devel' % entry['name']] for artifact in artifacts: artifact_uri = (base_uri + '/strata/' + entry['name'] + '/products/' + artifact) system.add(BASEROCK.containsStratumArtifact, rdflib.URIRef(artifact_uri)) system.set(BASEROCK.hasConfigurationExtension, rdflib.Literal( contents.get('configuration-extensions', []))) elif contents['kind'] == 'cluster': cluster_uri = base_uri + 'clusters/' + contents['name'] entity = cluster = rdflib.resource.Resource( graph, rdflib.URIRef(cluster_uri)) deployments = [] for entry in contents.get('systems', []): system_morph_file = os.path.join(toplevel_path, entry['morph']) system_name = get_name_from_morph_file(system_morph_file) system_uri = base_uri + 'systems/' + system_name # FIXME: ignores deploy-defaults at present for label, details in entry['deploy'].items(): deployment_uri = cluster_uri + '/' + label deployment = rdflib.resource.Resource( graph, rdflib.URIRef(deployment_uri)) deployment.set(BASEROCK.deploysSystem, rdflib.URIRef(system_uri)) deployment.set(BASEROCK.hasLabel, rdflib.Literal(label)) deployment.set(BASEROCK.hasType, rdflib.Literal(details['type'])) deployment.set(BASEROCK.hasLocation, rdflib.Literal(details['location'])) settings = [] for key, value in details.items(): if key in ['type', 'location']: continue # FIXME: RDF must have a way of representing arbitrary # key/values better than using a string with an = # sign... settings.append('%s=%s' % (key, value)) deployment.set(BASEROCK.hasConfigurationSetting, rdflib.Literal(settings)) cluster.add(BASEROCK.deploysSystem, deployment) if 'description' in contents: entity.set(DUBLIN_CORE.description, rdflib.Literal(contents['description'])) # FIXME: comments from the .yaml file are lost ... as a quick solution, # you could manually find every line from the YAML that starts with a # '#' and dump that into a property. Or ruamel.yaml might help? print 'Parsing .morph files...' for dirname, dirnames, filenames in os.walk(toplevel_path): if '.git' in dirnames: dirnames.remove('.git') for filename in sorted(filenames): if filename.endswith('.morph'): try: load_morph(toplevel_path, os.path.join(dirname, filename)) except Exception as e: print '%s: %r' % (filename, e) raise return graph