Factor scenario running logic out into yarnlib

There's now ScenarioRunner, which is given the environment to run
scenarios in, such as the shell prelude, environment variables and the
path to the source tree of the project being tested.

This is then given scenarios and the state directories.

POINTS: The callbacks led to some nasty contortions to keep the same
functionality, should run_scenario return some form of iterator, so the
control flow is fully under control of the caller.

3 files changed, 232 insertions, 104 deletions

diff --git a/yarn b/yarn
index 12e01dd..db18cd7 100755
--- a/yarn
+++ b/yarn
@@ -118,6 +118,13 @@ class YarnRunner(cliapp.Application):
                 '%String(scenario_name): '
                 '%String(step_name)')
 
+        if self.settings['tempdir']:
+            self.tempdir = os.path.abspath(self.settings['tempdir'])
+            if not os.path.exists(self.tempdir):
+                os.mkdir(self.tempdir)
+        else:
+            self.tempdir = tempfile.mkdtemp()
+
         scenarios, implementations = self.parse_scenarios(args)
         sv = yarnlib.ScenarioValidator(scenarios)
         sv.validate_all()
@@ -136,10 +143,15 @@ class YarnRunner(cliapp.Application):
         self.steps_run = 0
         self.timings = []
 
+        scenario_runner = yarnlib.ScenarioRunner(shell_prelude, os.getcwd(),
+                                                 self.parse_env(),
+                                                 pre_step_cb=self.pre_step,
+                                                 post_step_cb=self.post_step)
+
         start_time = time.time()
         failed_scenarios = []
         for scenario in self.select_scenarios(scenarios):
-            if not self.run_scenario(scenario, shell_prelude):
+            if not self.run_scenario(scenario_runner, scenario):
                 failed_scenarios.append(scenario)
         duration = time.time() - start_time
 
@@ -219,7 +231,7 @@ class YarnRunner(cliapp.Application):
 
         return scenarios
 
-    def run_scenario(self, scenario, shell_prelude):
+    def run_scenario(self, scenario_runner, scenario):
         self.start_scenario_timing(scenario.name)
         started = time.time()
 
@@ -234,62 +246,18 @@ class YarnRunner(cliapp.Application):
             self.remember_scenario_timing(time.time() - started)
             return True
 
-        if self.settings['tempdir']:
-            tempdir = os.path.abspath(self.settings['tempdir'])
-            if not os.path.exists(tempdir):
-                os.mkdir(tempdir)
-        else:
-            tempdir = tempfile.mkdtemp()
-
-        os.mkdir(self.scenario_dir(tempdir, scenario))
-        datadir = self.datadir(tempdir, scenario)
+        os.mkdir(self.scenario_dir(self.tempdir, scenario))
+        datadir = self.datadir(self.tempdir, scenario)
         os.mkdir(datadir)
         self.info('DATADIR is %s' % datadir)
         homedir = self.homedir(datadir)
         os.mkdir(homedir)
         self.info('HOME for tests is %s' % homedir)
 
-        assuming = [s for s in scenario.steps if s.what == 'ASSUMING']
-        cleanup = [s for s in scenario.steps if s.what == 'FINALLY']
-        normal = [s for s in scenario.steps if s not in assuming + cleanup]
-
-        ok = True
-        step_number = 0
-
-        for step in assuming:
-            exit = self.run_step(datadir, scenario, step, shell_prelude, False)
-            step_number += 1
-            self.snapshot_datadir(
-                tempdir, datadir, scenario, step_number, step)
-            if exit != 0:
-                self.ts.notify(
-                    'Skipping "%s" because "%s %s" failed' %
-                    (scenario.name, step.what, step.text))
-                self.skipped_for_assuming += 1
-                break
-        else:
-            for step in normal:
-                exit = self.run_step(
-                    datadir, scenario, step, shell_prelude, True)
-                step_number += 1
-                self.snapshot_datadir(
-                    tempdir, datadir, scenario, step_number, step)
-                if exit != 0:
-                    ok = False
-                    break
-
-            for step in cleanup:
-                exit = self.run_step(
-                    datadir, scenario, step, shell_prelude, True)
-                step_number += 1
-                self.snapshot_datadir(
-                    tempdir, datadir, scenario, step_number, step)
-                if exit != 0:
-                    ok = False
-                    break
+        ok = scenario_runner.run_scenario(scenario, datadir, homedir)
 
         if not self.settings['snapshot']:
-            shutil.rmtree(tempdir)
+            shutil.rmtree(self.tempdir)
 
         self.remember_scenario_timing(time.time() - started)
         return ok
@@ -297,42 +265,16 @@ class YarnRunner(cliapp.Application):
     def homedir(self, datadir):
         return os.path.join(datadir, 'HOME')
 
-    def clean_env(self):
-        '''Return a clean environment for running tests.'''
-
-        whitelisted = [
-            'PATH',
-        ]
-
-        hardcoded = {
-            'TERM': 'dumb',
-            'SHELL': '/bin/sh',
-            'LC_ALL': 'C',
-            'USER': 'tomjon',
-            'USERNAME': 'tomjon',
-            'LOGNAME': 'tomjon',
-        }
-
-        env = {}
-
-        for key in whitelisted:
-            if key in os.environ:
-                env[key] = os.environ[key]
-
-        for key in hardcoded:
-            env[key] = hardcoded[key]
-
+    def parse_env(self):
         for option_arg in self.settings['env']:
             if '=' not in option_arg:
                 raise cliapp.AppException(
                     '--env argument must contain "=" '
                     'to separate environment variable name and value')
             key, value = option_arg.split('=', 1)
-            env[key] = value
-
-        return env
+            yield key, value
 
-    def run_step(self, datadir, scenario, step, shell_prelude, report_error):
+    def pre_step(self, scenario, step, step_number, scenario_env):
         started = time.time()
 
         self.info('Running step "%s %s"' % (step.what, step.text))
@@ -340,19 +282,12 @@ class YarnRunner(cliapp.Application):
         self.ts['step_name'] = '%s %s' % (step.what, step.text)
         self.steps_run += 1
 
-        m = yarnlib.implements_matches_step(step.implementation, step)
-        assert m is not None
-        env = self.clean_env()
-        env['DATADIR'] = datadir
-        env['SRCDIR'] = os.getcwd()
-        env['HOME'] = self.homedir(datadir)
-        for i, match in enumerate(m.groups('')):
-            env['MATCH_%d' % (i+1)] = match
+        return (started,)
 
-        shell_script = '%s\n\n%s\n' % (
-            shell_prelude, step.implementation.shell)
-        exit, stdout, stderr = cliapp.runcmd_unchecked(
-            ['sh', '-xeuc', shell_script], env=env)
+    def post_step(self, scenario, step, step_number, scenario_env,
+                  exit, stdout, stderr, pre_step_userdata):
+        stopped = time.time()
+        (started,) = pre_step_userdata
 
         logging.debug('Exit code: %d' % exit)
         if stdout:
@@ -363,20 +298,24 @@ class YarnRunner(cliapp.Application):
             logging.debug('Standard error:\n%s' % self.indent(stderr))
         else:
             logging.debug('Standard error: empty')
-
-        if exit != 0 and report_error:
-            self.error(
-                'ERROR: In scenario "%s"\nstep "%s %s" failed,\n'
-                'with exit code %d:\n'
-                'Standard output from shell command:\n%s'
-                'Standard error from shell command:\n%s' %
-                (scenario.name, step.what, step.text, exit,
-                 self.indent(stdout), self.indent(stderr)))
-
+        if exit != 0:
+            if step.what == 'ASSUMING':
+                self.ts.notify(
+                    'Skipping "%s" because "%s %s" failed' %
+                    (scenario.name, step.what, step.text))
+                self.skipped_for_assuming += 1
+            else:
+                self.error(
+                    'ERROR: In scenario "%s"\nstep "%s %s" failed,\n'
+                    'with exit code %d:\n'
+                    'Standard output from shell command:\n%s'
+                    'Standard error from shell command:\n%s' %
+                    (scenario.name, step.what, step.text, exit,
+                     self.indent(stdout), self.indent(stderr)))
         self.remember_step_timing(
-            '%s %s' % (step.what, step.text), time.time() - started)
-
-        return exit
+            '%s %s' % (step.what, step.text), stopped - started)
+        self.snapshot_datadir(self.tempdir, scenario_env['DATADIR'],
+                              scenario, step_number, step)
 
     def scenario_dir(self, tempdir, scenario):
         return os.path.join(tempdir, self.nice(scenario.name))
diff --git a/yarnlib/__init__.py b/yarnlib/__init__.py
index 0d36668..f2d8381 100644
--- a/yarnlib/__init__.py
+++ b/yarnlib/__init__.py
@@ -25,6 +25,7 @@ from scenario_step_connector import (implements_matches_step,
                                      ScenarioStepConnector,
                                      StepNotImplementedError,
                                      StepMultipleImplementationsError)
+from scenario_runner import ScenarioRunner
 
 
 import shell_libraries
diff --git a/yarnlib/scenario_runner.py b/yarnlib/scenario_runner.py
new file mode 100644
index 0000000..272c57c
--- /dev/null
+++ b/yarnlib/scenario_runner.py
@@ -0,0 +1,188 @@
+# Copyright 2014  Codethink Limited
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import cliapp
+import os
+
+import yarnlib
+
+
+def default_pre_step(scenario, step, step_number, scenario_env):
+    '''Default callback run before each step in ScenarioRunner.
+
+    Parameters:
+
+    * `scenario`: The yarnlib.Scenario that is about to be run.
+    * `step`: The yarnlib.ScenarioStep that is about to be run.
+    * `step_number`: Per-scenario counter for the step that is about to be run,
+                     starting from 1.
+    * `step_env`: Environment that will be used for this step.
+    * `return`: Any returned values are passed to `post_step_cb`.
+
+    '''
+    pass
+
+
+def default_post_step(scenario, step, step_number, scenario_env,
+                      exit, stdout, stderr, pre_step_userdata):
+    '''Default callback run after each step in ScenarioRunner.
+
+    Parameters:
+
+    * `scenario`: The yarnlib.Scenario that has just been run.
+    * `step`: The yarnlib.ScenarioStep that has just been run.
+    * `step_number`: Per-scenario counter for the step that is about to be run,
+                     starting from 1.
+    * `step_env`: Environment that was used for this step.
+    * `exit`: Return code of the step that was run.
+    * `stdout`: Standard output of the step that was just run.
+    * `stderr`: Standard error of the step that was just run.
+    * `pre_step_userdata`: Return value from `pre_step_cb`.
+
+    '''
+    pass
+
+
+# Arguably this could just be the run_scenario method with all the
+# constructor's parameters passed in, but reeks of poor design with that
+# many parameters.
+# However, decoupling the ScenarioRunner from the scenarios allows the
+# same scenarios to be run against different projects that aim to satisfy
+# the same requirements, or multiple yarn suites on the same project,
+# if the project aims to satisfy multiple sets of requirements.
+# While this use is far less likely than having 1 yarn suite per project,
+# it's a useful metric for deciding how to split the arguments between the
+# constructor and the run_scenario method.
+class ScenarioRunner(object):
+    '''Sets up an environment to run scenarios on a Project.
+
+    Parameters:
+
+    * `shell_prelude`: Prefix to all shell scripts run, e.g. shell libraries.
+    * `srcdir`: Path to the root of the source tree.
+                This is available as the $SRCDIR environment variable
+                and is the directory the scenarios are run from.
+    * `extra_env`: dict, or iterable of pairs of variables to add to
+                   the clean environment provided for every command.
+    * `pre_step_cb`: Callback run before each step in a scenario.
+                     See default_pre_step for an explanation of its signature.
+                     Its return value is passed to `post_step_cb`.
+    * `post_step_cb`: Callback run after each step in a scenario.
+                     See default_post_step for an explanation of its signature.
+    * `cmdrunner`: Function used to run step commands in place of
+                   `cliapp.runcmd_unchecked`, intended to be replaceable
+                   by unit tests.
+
+    '''
+
+
+    def __init__(self, shell_prelude, srcdir, extra_env=(),
+                 pre_step_cb=default_pre_step, post_step_cb=default_post_step,
+                 cmdrunner=cliapp.runcmd_unchecked):
+        self.shell_prelude = shell_prelude
+        self.srcdir = srcdir
+        self.env = self.clean_env(extra_env, SRCDIR=srcdir)
+        self.pre_step_cb = pre_step_cb
+        self.post_step_cb = post_step_cb
+        self.cmdrunner = cmdrunner
+
+    def run_scenario(self, scenario, datadir, homedir):
+        assuming = [s for s in scenario.steps if s.what == 'ASSUMING']
+        cleanup = [s for s in scenario.steps if s.what == 'FINALLY']
+        normal = [s for s in scenario.steps if s not in assuming + cleanup]
+
+        ok = True
+        step_number = 1
+
+        scenario_env = dict(self.env)
+        scenario_env['HOME'] = homedir
+        scenario_env['DATADIR'] = datadir
+
+        for step in assuming:
+            exit = self.run_step(scenario, step, scenario_env, step_number)
+            step_number += 1
+            if exit != 0:
+                break
+        else:
+            for step in normal:
+                exit = self.run_step(scenario, step, scenario_env, step_number)
+                step_number += 1
+                if exit != 0:
+                    ok = False
+                    break
+
+            for step in cleanup:
+                exit = self.run_step(scenario, step, scenario_env, step_number)
+                step_number += 1
+                if exit != 0:
+                    ok = False
+                    break
+
+        return ok
+
+    def run_step(self, scenario, step, scenario_env, step_number):
+        m = yarnlib.implements_matches_step(step.implementation, step)
+        assert m is not None
+        step_env = dict(scenario_env)
+        for i, match in enumerate(m.groups('')):
+            step_env['MATCH_%d' % (i+1)] = match
+
+        cb_userdata = self.pre_step_cb(scenario, step, step_number,
+                                       step_env)
+
+        shell_script = '%s\n\n%s\n' % (
+            self.shell_prelude, step.implementation.shell)
+        exit, stdout, stderr = self.cmdrunner(
+            ['sh', '-xeuc', shell_script], env=step_env, cwd=self.srcdir)
+
+        self.post_step_cb(scenario, step, step_number, step_env,
+                          exit, stdout, stderr, cb_userdata)
+
+        return exit
+
+    @staticmethod
+    def clean_env(extra_env, **kwarg_env):
+        '''Return a clean environment for running tests.'''
+
+        whitelisted = [
+            'PATH',
+        ]
+
+        hardcoded = {
+            'TERM': 'dumb',
+            'SHELL': '/bin/sh',
+            'LC_ALL': 'C',
+            'USER': 'tomjon',
+            'USERNAME': 'tomjon',
+            'LOGNAME': 'tomjon',
+        }
+
+        env = {}
+
+        for key in whitelisted:
+            if key in os.environ:
+                env[key] = os.environ[key]
+
+        env.update(hardcoded)
+        env.update(extra_env)
+        env.update(kwarg_env)
+
+        return env
+
+
+# vim: set ts=4 sw=4 et: