# Load Baserock Definitions serialisation format V5 into a SurfRDF 'store'. # https://github.com/cosminbasca/surfrdf import rdflib import surf import yaml import os import warnings surf.ns.register(baserock='http://baserock.org/definitions/example-schema#') surf.ns.register(dc='http://purl.org/dc/terms/') def load_all_morphologies(session, store): Chunk = session.get_class(surf.ns.BASEROCK.Chunk) ChunkReference = session.get_class(surf.ns.BASEROCK.ChunkReference) Stratum = session.get_class(surf.ns.BASEROCK.Stratum) StratumArtifact = session.get_class(surf.ns.BASEROCK.StratumArtifact) System = session.get_class(surf.ns.BASEROCK.System) SystemDeployment = session.get_class(surf.ns.BASEROCK.SystemDeployment) Cluster = session.get_class(surf.ns.BASEROCK.Cluster) def load_morph(path): try: with open(path) as f: text = f.read() contents = yaml.safe_load(text) morph_type = contents['kind'] assert 'name' in contents assert contents['kind'] in ['cluster', 'system', 'stratum', 'chunk'] except Exception as e: warnings.warn("Problem loading %s: %s" % (path, e)) # FIXME: base_uri = 'http://example.com/' entity = None # Note the 'surf' library doesn't seem to do any kind of validity checking # so you can insert whatever random data you feel like, if you want. if contents['kind'] == 'chunk': chunk_uri = base_uri + 'chunks/' + contents['name'] entity = chunk = Chunk(chunk_uri) # FIXME: I think order is lost here !!!!! if 'pre-configure-commands' in contents: chunk.baserock_preConfigureCommands = contents['pre-configure-commands'] if 'configure-commands' in contents: chunk.baserock_configureCommands = contents['configure-commands'] if 'post-configure-commands' in contents: chunk.baserock_postConfigureCommands = contents['post-configure-commands'] if 'pre-build-commands' in contents: chunk.baserock_preBuildCommands = contents['pre-build-commands'] if 'build-commands' in contents: chunk.baserock_buildCommands = contents['build-commands'] if 'post-build-commands' in contents: chunk.baserock_postBuildCommands = contents['post-build-commands'] if 'pre-install-commands' in contents: chunk.baserock_preInstallCommands = contents['pre-install-commands'] if 'install-commands' in contents: chunk.baserock_installCommands = contents['install-commands'] if 'post-install-commands' in contents: chunk.baserock_postInstallCommands = contents['post-install-commands'] elif contents['kind'] == 'stratum': stratum_uri = base_uri + 'strata/' + contents['name'] entity = stratum = Stratum(stratum_uri) stratum_build_deps = [] for entry in contents.get('build-depends', []): build_dep_uri = base_uri + 'strata/' + entry['morph'] stratum_build_deps.append(rdflib.URIRef(build_dep_uri)) stratum.baserock_hasBuildDependency = stratum_build_deps artifacts = [] for entry in contents.get('products', []): artifact_uri = stratum_uri + '/products/' + entry['artifact'] artifact = StratumArtifact(artifact_uri) # FIXME: order probably lost here if 'includes' in entry: artifact.baserock_includes = entry['includes'] artifacts.append(artifact) stratum.baserock_produces = artifacts chunk_refs = [] for entry in contents.get('chunks', []): chunk_ref_uri = stratum_uri + '/chunk-refs/' + entry['name'] chunk_ref = ChunkReference(chunk_ref_uri) # FIXME: this ignores the 'morph' field, and assumes 'name' is # usable as-is. chunk_uri = base_uri + 'chunks/' + entry['name'] chunk_ref.baserock_refersToChunk = rdflib.URIRef(chunk_uri) chunk_ref.baserock_repo = entry['repo'] chunk_ref.baserock_ref = entry['ref'] if 'unpetrify-ref' in entry: chunk_ref.baserock_unpetrifyRef = entry['unpetrify-ref'] chunk_ref.baserock_buildMode = entry.get('build-mode', 'normal') chunk_ref.baserock_prefix = entry.get('prefix', '/usr') chunk_ref_build_deps = [] for entry_dep in entry.get('build-depends', []): build_dep_uri = stratum_uri + '/chunk-refs/' + entry_dep chunk_ref_build_deps.append(build_dep_uri) chunk_ref.baserock_hasChunkBuildDependency = chunk_ref_build_deps chunk_refs.append(chunk_ref) stratum.baserock_containsChunkReference = chunk_refs elif contents['kind'] == 'system': system_uri = base_uri + 'systems/' + contents['name'] entity = system = System(system_uri) system.baserock_arch = contents['arch'] stratum_artifacts = [] for entry in contents.get('strata', []): # FIXME: need to include all strata if 'artifacts' isn't specified, # which is difficult becausee they might not all be loaded yet ... # so for now I cheat and just assume -runtime and -devel. If there # are extra artifacts for the stratum they won't be incuded by # default. I'm not sure if this is how Morph behaves or not. artifacts = entry.get('artifacts') if artifacts is None: artifacts = ['%s-runtime' % entry['name'], '%s-devel' % entry['name']] for artifact in artifacts: artifact_uri = (base_uri + '/strata/' + entry['name'] + '/products/' + artifact) stratum_artifacts.append(artifact_uri) system.baserock_containsStratumArtifact = stratum_artifacts system.baserock_hasConfigurationExtension = \ contents.get('configuration-extensions', []) elif contents['kind'] == 'cluster': cluster_uri = base_uri + 'clusters/' + contents['name'] entity = cluster = Cluster(cluster_uri) deployments = [] for entry in contents.get('systems', []): # FIXME: can't get the URI from the 'morph' entry... need to load # the actual .morph file and get the name from there. system_uri = 'http://FIXME' # FIXME: ignores deploy-defaults at present for label, details in entry['deploy'].items(): deployment_uri = cluster_uri + '/' + label deployment = SystemDeployment(deployment_uri) deployment.baserock_deploysSystem = rdflib.URIRef(system_uri) deployment.baserock_hasLabel = label deployment.baserock_hasType = details['type'] deployment.baserock_hasLocation = details['location'] settings = [] for key, value in details.items(): if key in ['type', 'location']: continue # FIXME: RDF must have a way of representing arbitrary # key/values better than using a string with an = sign... settings.append('%s=%s' % (key,value)) deployment.baserock_hasConfigurationSetting = settings deployments.append(deployment) cluster.baserock_deploysSystem = deployments if 'description' in contents: entity.dc_description = contents['description'] # FIXME: is this needed? why? entity.set_dirty(True) # FIXME: comments from the .yaml file are lost ... as a quick solution, # you could manually find every line from the YAML that starts with a '#' # and dump that into a property. print 'Parsing .morph files...' for dirname, dirnames, filenames in os.walk('..'): if '.git' in dirnames: dirnames.remove('.git') for filename in sorted(filenames): if filename.endswith('.morph'): load_morph(os.path.join(dirname, filename)) print 'Committing to database...' try: session.commit() except Exception as e: if 'Virtuoso 42000 Error SR186: No permission to execute procedure DB' in e.message: print("Permission denied trying to update the database via the " "SPARQL endpoint. By default this endpoint is read-only. " "To enable write access, run the `isql` or `isql-vt` " "commandline interface and run the following statement:\n\n" " grant SPARQL_UPDATE to \"SPARQL\";\n\n" "WARNING! Only do this if you're using a local test instance " "of Virtuoso. You need to set up a real authenticated user " "account if using an instance of Virtuoso that has " "important data on it.") else: raise store.save()