diff options
Diffstat (limited to 'schema/parse.py')
| -rw-r--r-- | schema/parse.py | 210 |
1 files changed, 210 insertions, 0 deletions
diff --git a/schema/parse.py b/schema/parse.py new file mode 100644 index 00000000..a118e70b --- /dev/null +++ b/schema/parse.py @@ -0,0 +1,210 @@ +# Load Baserock Definitions serialisation format V5 into a SurfRDF 'store'. + +# https://github.com/cosminbasca/surfrdf + + +import rdflib +import surf +import yaml + +import os +import warnings + + +surf.ns.register(baserock='http://baserock.org/definitions/example-schema#') + + +def load_all_morphologies(session, store): + Chunk = session.get_class(surf.ns.BASEROCK.Chunk) + ChunkReference = session.get_class(surf.ns.BASEROCK.ChunkReference) + Stratum = session.get_class(surf.ns.BASEROCK.Stratum) + StratumArtifact = session.get_class(surf.ns.BASEROCK.StratumArtifact) + System = session.get_class(surf.ns.BASEROCK.System) + SystemDeployment = session.get_class(surf.ns.BASEROCK.SystemDeployment) + Cluster = session.get_class(surf.ns.BASEROCK.Cluster) + + def load_morph(path): + try: + with open(path) as f: + text = f.read() + contents = yaml.safe_load(text) + morph_type = contents['kind'] + assert 'name' in contents + assert contents['kind'] in ['cluster', 'system', 'stratum', 'chunk'] + except Exception as e: + warnings.warn("Problem loading %s: %s" % (path, e)) + + # FIXME: + base_uri = 'http://example.com/' + + entity = None + + # Note the 'surf' library doesn't seem to do any kind of validity checking + # so you can insert whatever random data you feel like, if you want. + + if contents['kind'] == 'chunk': + chunk_uri = base_uri + 'chunks/' + contents['name'] + entity = chunk = Chunk(chunk_uri) + + # FIXME: I think order is lost here !!!!! + if 'pre-configure-commands' in contents: + chunk.baserock_preConfigureCommands = contents['pre-configure-commands'] + if 'configure-commands' in contents: + chunk.baserock_configureCommands = contents['configure-commands'] + if 'post-configure-commands' in contents: + chunk.baserock_postConfigureCommands = contents['post-configure-commands'] + if 'pre-build-commands' in contents: + chunk.baserock_preBuildCommands = contents['pre-build-commands'] + if 'build-commands' in contents: + chunk.baserock_buildCommands = contents['build-commands'] + if 'post-build-commands' in contents: + chunk.baserock_postBuildCommands = contents['post-build-commands'] + if 'pre-install-commands' in contents: + chunk.baserock_preInstallCommands = contents['pre-install-commands'] + if 'install-commands' in contents: + chunk.baserock_installCommands = contents['install-commands'] + if 'post-install-commands' in contents: + chunk.baserock_postInstallCommands = contents['post-install-commands'] + + elif contents['kind'] == 'stratum': + stratum_uri = base_uri + 'strata/' + contents['name'] + entity = stratum = Stratum(stratum_uri) + + stratum_build_deps = [] + for entry in contents.get('build-depends', []): + build_dep_uri = base_uri + 'strata/' + entry['morph'] + stratum_build_deps.append(rdflib.URIRef(build_dep_uri)) + stratum.baserock_hasBuildDependency = stratum_build_deps + + artifacts = [] + for entry in contents.get('products', []): + artifact_uri = stratum_uri + '/products/' + entry['artifact'] + artifact = StratumArtifact(artifact_uri) + # FIXME: order probably lost here + if 'includes' in entry: + artifact.baserock_includes = entry['includes'] + artifacts.append(artifact) + stratum.baserock_produces = artifacts + + chunk_refs = [] + for entry in contents.get('chunks', []): + chunk_ref_uri = stratum_uri + '/chunk-refs/' + entry['name'] + chunk_ref = ChunkReference(chunk_ref_uri) + + # FIXME: this ignores the 'morph' field, and assumes 'name' is + # usable as-is. + chunk_uri = base_uri + 'chunks/' + entry['name'] + chunk_ref.baserock_refersToChunk = rdflib.URIRef(chunk_uri) + + chunk_ref.baserock_repo = entry['repo'] + chunk_ref.baserock_ref = entry['ref'] + if 'unpetrify-ref' in entry: + chunk_ref.baserock_unpetrifyRef = entry['unpetrify-ref'] + chunk_ref.baserock_buildMode = entry.get('build-mode', 'normal') + chunk_ref.baserock_prefix = entry.get('prefix', '/usr') + + chunk_ref_build_deps = [] + for entry_dep in entry.get('build-depends', []): + build_dep_uri = stratum_uri + '/chunk-refs/' + entry_dep + chunk_ref_build_deps.append(build_dep_uri) + chunk_ref.baserock_hasChunkBuildDependency = chunk_ref_build_deps + + chunk_refs.append(chunk_ref) + + stratum.baserock_containsChunkReference = chunk_refs + + elif contents['kind'] == 'system': + system_uri = base_uri + 'systems/' + contents['name'] + entity = system = System(system_uri) + + system.baserock_arch = contents['arch'] + + stratum_artifacts = [] + for entry in contents.get('strata', []): + # FIXME: need to include all strata if 'artifacts' isn't specified, + # which is difficult becausee they might not all be loaded yet ... + # so for now I cheat and just assume -runtime and -devel. If there + # are extra artifacts for the stratum they won't be incuded by + # default. I'm not sure if this is how Morph behaves or not. + artifacts = entry.get('artifacts') + if artifacts is None: + artifacts = ['%s-runtime' % entry['name'], + '%s-devel' % entry['name']] + for artifact in artifacts: + artifact_uri = (base_uri + '/strata/' + entry['name'] + + '/products/' + artifact) + stratum_artifacts.append(artifact_uri) + system.baserock_containsStratumArtifact = stratum_artifacts + + system.baserock_hasConfigurationExtension = \ + contents.get('configuration-extensions', []) + + elif contents['kind'] == 'cluster': + cluster_uri = base_uri + 'clusters/' + contents['name'] + entity = cluster = Cluster(cluster_uri) + + deployments = [] + for entry in contents.get('systems', []): + # FIXME: can't get the URI from the 'morph' entry... need to load + # the actual .morph file and get the name from there. + system_uri = 'http://FIXME' + + # FIXME: ignores deploy-defaults at present + for label, details in entry['deploy'].items(): + deployment_uri = cluster_uri + '/' + label + deployment = SystemDeployment(deployment_uri) + + deployment.baserock_deploysSystem = rdflib.URIRef(system_uri) + deployment.baserock_hasLabel = label + + deployment.baserock_hasType = details['type'] + deployment.baserock_hasLocation = details['location'] + + settings = [] + for key, value in details.items(): + if key in ['type', 'location']: + continue + # FIXME: RDF must have a way of representing arbitrary + # key/values better than using a string with an = sign... + settings.append('%s=%s' % (key,value)) + deployment.baserock_hasConfigurationSetting = settings + deployments.append(deployment) + + cluster.baserock_deploysSystem = deployments + + if 'description' in contents: + entity.baserock_description = contents['description'] + + # FIXME: is this needed? why? + entity.set_dirty(True) + # FIXME: comments from the .yaml file are lost ... as a quick solution, + # you could manually find every line from the YAML that starts with a '#' + # and dump that into a property. + + print 'Parsing .morph files...' + for dirname, dirnames, filenames in os.walk('..'): + if '.git' in dirnames: + dirnames.remove('.git') + for filename in sorted(filenames): + if filename.endswith('.morph'): + load_morph(os.path.join(dirname, filename)) + + print 'Committing to database...' + try: + session.commit() + except Exception as e: + if DATABASE=='virtuoso' and \ + 'Virtuoso 42000 Error SR186: No permission to execute procedure DB' in e.message: + print("Permission denied trying to update the database via the " + "SPARQL endpoint. By default this endpoint is read-only. " + "To enable write access, run the `isql` or `isql-vt` " + "commandline interface and run the following statement:\n\n" + " grant SPARQL_UPDATE to \"SPARQL\";\n\n" + "WARNING! Only do this if you're using a local test instance " + "of Virtuoso. You need to set up a real authenticated user " + "account if using an instance of Virtuoso that has " + "important data on it.") + else: + raise + + store.save() |