deploy: Record deployment information in deployed system

This commit introduces a new requirement: USERS MUST NOT HAVE SENSITIVE
DATA IN THEIR ENVIRONMENT. Otherwise it will be leaked into the system.
Note that configuration fields with 'PASSWORD' in their name are
stripped before writing the /baserock/deployment.meta file, so the
OpenStack OS_PASSWORD field is not leaked.

We want this so that we can run hooks at upgrade-time in the future.
These hooks might need to know how the system was configured and what
releaseuu it was. I'm not quite sure how we will define 'release' yet,
but by using `git tag` and `git describe` we are able to textually label
a time period in the history of the system's source code. We already
have the specific SHA1 of definitions.git stored in the system metadata,
so this should give us enough to be able to implement specific hooks
that work around any awkward upgrade complications we encounter in the
future.

4 files changed, 86 insertions, 3 deletions

diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index be2137b2..f5ef0061 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Codethink Limited
+# Copyright (C) 2013-2014  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -610,6 +610,11 @@ class GitDirectory(object):
         except Exception, e:
             raise RefDeleteError(self, ref, old_sha1, e)
 
+    def describe(self):
+        version = self._runcmd(
+            ['git', 'describe', '--always', '--dirty=-unreproducible'])
+        return version.strip()
+
 
 def init(dirname):
     '''Initialise a new git repository.'''
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index 21a6b5b8..8c312c1b 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Codethink Limited
+# Copyright (C) 2013-2014  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -202,6 +202,15 @@ class GitDirectoryContentsTests(unittest.TestCase):
                                 )
         self.assertEqual(expected, gd.get_commit_contents(commit).split('\n'))
 
+    def test_describe(self):
+        gd = morphlib.gitdir.GitDirectory(self.dirname)
+
+        gd._runcmd(['git', 'tag', '-a', '-m', 'Example', 'example', 'HEAD'])
+        self.assertEqual(gd.describe(), 'example-unreproducible')
+
+        gd._runcmd(['git', 'reset', '--hard'])
+        self.assertEqual(gd.describe(), 'example')
+
 
 class GitDirectoryRefTwiddlingTests(unittest.TestCase):
 
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index c8538a8d..61e1b5a4 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -16,6 +16,7 @@
 
 import cliapp
 import contextlib
+import json
 import os
 import shutil
 import stat
@@ -261,6 +262,13 @@ class DeployPlugin(cliapp.Plugin):
         are set as environment variables when either the configuration or the
         write extension runs (except `type` and `location`).
 
+        Deployment configuration is stored in the deployed system as
+        /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET
+        DURINGR DEPLOYMENT, so make sure you have no sensitive information in
+        your environment that is being leaked. As a special case, any
+        environment/deployment variable that contains 'PASSWORD' in its name is
+        stripped out and not stored in the final system.
+
         '''
 
         if not args:
@@ -419,6 +427,14 @@ class DeployPlugin(cliapp.Plugin):
                 msg='System unpacked at %(system_tree)s',
                 system_tree=system_tree)
 
+            self.app.status(
+                msg='Writing deployment metadata file')
+            metadata = self.create_metadata(
+                    artifact, root_repo_dir, deployment_type, location, env)
+            metadata_path = os.path.join(
+                    system_tree, 'baserock', 'deployment.meta')
+            with morphlib.savefile.SaveFile(metadata_path, 'w') as f:
+                f.write(json.dumps(metadata, indent=4, sort_keys=True))
 
             # Run configuration extensions.
             self.app.status(msg='Configure system')
@@ -496,3 +512,41 @@ class DeployPlugin(cliapp.Plugin):
         st = os.stat(filename)
         mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
         return (stat.S_IMODE(st.st_mode) & mask) != 0
+
+    def create_metadata(self, system_artifact, root_repo_dir, deployment_type,
+                        location, env):
+        '''Deployment-specific metadata.
+
+        The `build` and `deploy` operations must be from the same ref, so full
+        info on the root repo that the system came from is in
+        /baserock/${system_artifact}.meta and is not duplicated here. We do
+        store a `git describe` of the definitions.git repo as a convenience for
+        post-upgrade hooks that we may need to implement at a future date:
+        the `git describe` output lists the last tag, which will hopefully help
+        us to identify which release of a system was deployed without having to
+        keep a list of SHA1s somewhere or query a Trove.
+
+        '''
+
+        def remove_passwords(env):
+            def is_password(key):
+                return 'PASSWORD' in key
+            return { k:v for k, v in env.iteritems() if not is_password(k) }
+
+        meta = {
+            'system-artifact-name': system_artifact.name,
+            'configuration': remove_passwords(env),
+            'deployment-type': deployment_type,
+            'location': location,
+            'definitions-version': {
+                'describe': root_repo_dir.describe(),
+            },
+            'morph-version': {
+                'ref': morphlib.gitversion.ref,
+                'tree': morphlib.gitversion.tree,
+                'commit': morphlib.gitversion.commit,
+                'version': morphlib.gitversion.version,
+            },
+        }
+
+        return meta
diff --git a/tests.deploy/deploy-cluster.script b/tests.deploy/deploy-cluster.script
index 0efc8d3c..3ef60479 100755
--- a/tests.deploy/deploy-cluster.script
+++ b/tests.deploy/deploy-cluster.script
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (C) 2013  Codethink Limited
+# Copyright (C) 2013-2014  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -29,8 +29,11 @@ cd "$DATADIR/workspace/branch1"
 
 "$SRCDIR/scripts/test-morph" build linux-system
 
+GIT_DIR=test:morphs/.git git tag -a my-test-tag -m "Example tag" HEAD
+
 "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \
     deploy test_cluster \
+    linux-system-2.EXAMPLE_PASSWORD="secret" \
     linux-system-2.HOSTNAME="baserock-rocks-even-more" \
     > /dev/null
 
@@ -44,3 +47,15 @@ hostname2=$(tar -xf $outputdir/linux-system-2.tar ./etc/hostname -O)
 
 [ "$hostname1" = baserock-rocks ]
 [ "$hostname2" = baserock-rocks-even-more ]
+
+tar -xf $outputdir/linux-system-2.tar ./baserock/deployment.meta
+metadata=baserock/deployment.meta
+
+# Check that 'git describe' of definitions repo was stored correctly
+echo -n "definitions-version: "
+"$SRCDIR/scripts/yaml-extract" $metadata definitions-version
+
+echo -n "configuration.HOSTNAME: "
+"$SRCDIR/scripts/yaml-extract" $metadata configuration HOSTNAME
+
+! (grep -q "EXAMPLE_PASSWORD" $metadata)