From 11894bcd780ff0134b142c34c8c7fbf421c4d921 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Jan 2014 14:46:19 +0000 Subject: yarns: install misc files in /etc, not /usr/etc --- yarns/implementations.yarn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 5b9b39df..89f30c3e 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -238,7 +238,7 @@ another to hold a chunk. - | for cfgfile in test.conf README; do - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/etc/test.d/$cfgfile" + install -D -m 644 /dev/null "$DESTDIR/etc/test.d/$cfgfile" done EOF -- cgit v1.2.1 From 52e2b4aa1463e0247b6d1ef1653bbacc5dc0e02c Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Jan 2014 13:39:34 +0000 Subject: yarns: Move system artifact inclusion implementation This moves the GIVEN system $system uses $artifacts from $source to the generic implements section, and the Python implementation into the edit-morph helper script. --- scripts/edit-morph | 29 ++++++++++++++++++++++++++++- yarns/implementations.yarn | 12 ++++++++++++ yarns/splitting.yarn | 17 ++--------------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/scripts/edit-morph b/scripts/edit-morph index 967bbc13..acb33b7e 100755 --- a/scripts/edit-morph +++ b/scripts/edit-morph @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ import cliapp import os import re +import yaml import morphlib @@ -200,4 +201,30 @@ class EditMorph(cliapp.Application): return result + def cmd_set_system_artifact_depends(self, args): + '''Change the artifacts used by a System. + + Usage: MORPHOLOGY_FILE STRATUM_NAME ARTIFACTS + + ARTIFACTS is an English language string describing which artifacts + to include, since the primary use of this command is to assist + yarn tests. + + Example: edit-morph set-system-artifact-depends system.morph \ + build-essential "build-essential-minimal, + build-essential-runtime and build-essential-devel" + + ''' + + file_path = args[0] + stratum_name = args[1] + artifacts = re.split(r"\s+and\s+|,?\s*", args[2]) + with open(file_path, "r") as f: + d = yaml.load(f) + for spec in d["strata"]: + if spec.get("alias", spec["name"]) == stratum_name: + spec["artifacts"] = artifacts + with open(file_path, "w") as f: + yaml.dump(d, f) + EditMorph().run() diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 89f30c3e..7e9b7275 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -797,3 +797,15 @@ Implementations for tarball inspection IMPLEMENTS THEN tarball (\S+) doesn't contain (.*) ! tar -tf "$DATADIR/$MATCH_1" | grep -Fe "$MATCH_2" + +Implementations for morphology manipulation +========================================== + +Altering morphologies in their source repositories +-------------------------------------------------- + + IMPLEMENTS GIVEN system (\S+) uses (.+) from (\S+) + "$SRCDIR/scripts/edit-morph" set-system-artifact-depends \ + "$DATADIR/gits/morphs/$MATCH_1.morph" "$MATCH_3" "$MATCH_2" + run_in "$DATADIR/gits/morphs" git add "$MATCH_1.morph" + run_in "$DATADIR/gits/morphs" git commit -m "Make $MATCH_1 only use $MATCH_2" diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index e55d5ed0..40da5fb4 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -40,7 +40,7 @@ libraries. The only change we need to make is to add a field to the system morphology to select which artifact to use in the system. - AND system test-system only uses test-stratum-runtime from test-stratum + AND system test-system uses test-stratum-runtime from test-stratum WHEN the user checks out the system branch called master The best way to test that only using some stratum artifacts works is @@ -75,7 +75,7 @@ so we know that if the system successfully builds, then we only built chunks that were needed. AND stratum test-stratum has chunks that aren't used in test-stratum-minimal - AND system test-system only uses test-stratum-minimal from test-stratum + AND system test-system uses test-stratum-minimal from test-stratum WHEN the user checks out the system branch called master THEN morph build the system test-system of the branch master @@ -168,19 +168,6 @@ Implementations run_in "$DATADIR/gits/morphs" git add "$MATCH_1.morph" run_in "$DATADIR/gits/morphs" git commit -m 'Add default splitting rules' - IMPLEMENTS GIVEN system (\S+) only uses (\S+) from (\S+) - python -c 'import sys, yaml - with open(sys.argv[1], "r") as f: - d = yaml.load(f) - for spec in d["strata"]: - if spec["name"] == sys.argv[3]: - spec["artifacts"] = [sys.argv[2]] - with open(sys.argv[1], "w") as f: - yaml.dump(d, f) - ' "$DATADIR/gits/morphs/$MATCH_1.morph" "$MATCH_2" "$MATCH_3" - run_in "$DATADIR/gits/morphs" git add "$MATCH_1.morph" - run_in "$DATADIR/gits/morphs" git commit -m "Make $MATCH_1 only use $MATCH_2" - IMPLEMENTS GIVEN stratum (\S+) has chunks that aren't used in (\1-\S+) # Create an extra chunk that will never successfully build cat >"$DATADIR/gits/test-chunk/unbuildable-chunk.morph" < Date: Wed, 22 Jan 2014 14:48:29 +0000 Subject: yarns: Add regression test for cache-key clash --- scripts/edit-morph | 18 ++++++++++++++++++ yarns/implementations.yarn | 15 +++++++++++++++ yarns/regression.yarn | 19 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/scripts/edit-morph b/scripts/edit-morph index acb33b7e..2970cc6e 100755 --- a/scripts/edit-morph +++ b/scripts/edit-morph @@ -227,4 +227,22 @@ class EditMorph(cliapp.Application): with open(file_path, "w") as f: yaml.dump(d, f) + def cmd_set_stratum_match_rules(self, (file_path, match_rules)): + '''Set a stratum's match rules. + + Usage: FILE_PATH MATCH_RULES_YAML + + This sets the stratum's "products" field, which is used to + determine which chunk artifacts go into which stratum artifacts + the stratum produces. + + The match rules must be a string that yaml can parse. + + ''' + with open(file_path, "r") as f: + d = yaml.load(f) + d['products'] = yaml.load(match_rules) + with open(file_path, "w") as f: + yaml.dump(d, f) + EditMorph().run() diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 7e9b7275..b4d8e7ce 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -809,3 +809,18 @@ Altering morphologies in their source repositories "$DATADIR/gits/morphs/$MATCH_1.morph" "$MATCH_3" "$MATCH_2" run_in "$DATADIR/gits/morphs" git add "$MATCH_1.morph" run_in "$DATADIR/gits/morphs" git commit -m "Make $MATCH_1 only use $MATCH_2" + + IMPLEMENTS GIVEN stratum (\S+) has match rules: (.*) + cd "$DATADIR/gits/morphs" + "$SRCDIR/scripts/edit-morph" set-stratum-match-rules \ + "$MATCH_1.morph" "$MATCH_2" + git add "$MATCH_1.morph" + git commit -m "Make $MATCH_1 match $MATCH_2" + +Altering morphologies in the workspace +-------------------------------------- + + IMPLEMENTS GIVEN stratum (\S+) in system branch (\S+) has match rules: (.*) + cd "$DATADIR/workspace/$MATCH_2/test:morphs" + "$SRCDIR/scripts/edit-morph" set-stratum-match-rules \ + "$MATCH_1.morph" "$MATCH_3" diff --git a/yarns/regression.yarn b/yarns/regression.yarn index 49c663ec..71431ff7 100644 --- a/yarns/regression.yarn +++ b/yarns/regression.yarn @@ -46,6 +46,25 @@ area, hence their results cannot be trusted. AND the user attempts to build the system bootstrap-system in branch master THEN the build error message includes the string "No non-bootstrap chunks found" +When we started allowing multiple artifacts, a long-standing bug in +cache-key computation was discovered, it didn't include artifact names, +which would cause a collision if a morphology changed which artifacts +from a source it depended on, but not the number of artifacts from that +source it depended on. + + SCENARIO changing the artifacts a system uses + GIVEN a workspace + AND a git server + AND system test-system uses test-stratum-runtime from test-stratum + AND stratum test-stratum has match rules: [{artifact: test-stratum-runtime, include: [.*-(bins|libs|locale)]}, {artifact: test-stratum-devel, include: [.*-(devel|doc|misc)]}] + WHEN the user checks out the system branch called master + GIVEN a cluster called test-cluster for deploying only the test-system system as type tar in system branch master + WHEN the user builds the system test-system in branch master + GIVEN stratum test-stratum in system branch master has match rules: [{artifact: test-stratum-runtime, include: [.*-(bins|libs|misc)]}, {artifact: test-stratum-devel, include: [.*-(devel|doc|locale)]}] + WHEN the user builds the system test-system in branch master + AND the user deploys the cluster test-cluster in branch master with options system.location="$DATADIR/test.tar" + THEN tarball test.tar contains baserock/test-chunk-misc.meta + Implementations --------------- -- cgit v1.2.1