Factor scenario running logic out into yarnlibbaserock/richardmaw/S10275/factor-out-modules-v3

There's now ScenarioRunner, which is given the environment to run
scenarios in, such as the shell prelude, environment variables and the
path to the source tree of the project being tested.

This is then given scenarios and the state directories.

4 files changed, 377 insertions, 106 deletions

diff --git a/yarn b/yarn
index 131ea2d..6dc09c5 100755
--- a/yarn
+++ b/yarn
@@ -118,6 +118,13 @@ class YarnRunner(cliapp.Application):
                 '%String(scenario_name): '
                 '%String(step_name)')
 
+        if self.settings['tempdir']:
+            self.tempdir = os.path.abspath(self.settings['tempdir'])
+            if not os.path.exists(self.tempdir):
+                os.mkdir(self.tempdir)
+        else:
+            self.tempdir = tempfile.mkdtemp()
+
         scenarios, implementations = self.parse_scenarios(args)
         sv = yarnlib.ScenarioValidator(scenarios)
         sv.validate_all()
@@ -136,13 +143,21 @@ class YarnRunner(cliapp.Application):
         self.steps_run = 0
         self.timings = []
 
+        scenario_runner = yarnlib.ScenarioRunner(shell_prelude, os.getcwd(),
+                                                 self.parse_env(),
+                                                 pre_step_cb=self.pre_step,
+                                                 post_step_cb=self.post_step)
+
         start_time = time.time()
         failed_scenarios = []
         for scenario in self.select_scenarios(scenarios):
-            if not self.run_scenario(scenario, shell_prelude):
+            if not self.run_scenario(scenario_runner, scenario):
                 failed_scenarios.append(scenario)
         duration = time.time() - start_time
 
+        if not self.settings['snapshot']:
+            shutil.rmtree(self.tempdir)
+
         if not self.settings['quiet']:
             self.ts.clear()
             self.ts.finish()
@@ -219,7 +234,7 @@ class YarnRunner(cliapp.Application):
 
         return scenarios
 
-    def run_scenario(self, scenario, shell_prelude):
+    def run_scenario(self, scenario_runner, scenario):
         self.start_scenario_timing(scenario.name)
         started = time.time()
 
@@ -234,62 +249,15 @@ class YarnRunner(cliapp.Application):
             self.remember_scenario_timing(time.time() - started)
             return True
 
-        if self.settings['tempdir']:
-            tempdir = os.path.abspath(self.settings['tempdir'])
-            if not os.path.exists(tempdir):
-                os.mkdir(tempdir)
-        else:
-            tempdir = tempfile.mkdtemp()
-
-        os.mkdir(self.scenario_dir(tempdir, scenario))
-        datadir = self.datadir(tempdir, scenario)
+        os.mkdir(self.scenario_dir(self.tempdir, scenario))
+        datadir = self.datadir(self.tempdir, scenario)
         os.mkdir(datadir)
         self.info('DATADIR is %s' % datadir)
         homedir = self.homedir(datadir)
         os.mkdir(homedir)
         self.info('HOME for tests is %s' % homedir)
 
-        assuming = [s for s in scenario.steps if s.what == 'ASSUMING']
-        cleanup = [s for s in scenario.steps if s.what == 'FINALLY']
-        normal = [s for s in scenario.steps if s not in assuming + cleanup]
-
-        ok = True
-        step_number = 0
-
-        for step in assuming:
-            exit = self.run_step(datadir, scenario, step, shell_prelude, False)
-            step_number += 1
-            self.snapshot_datadir(
-                tempdir, datadir, scenario, step_number, step)
-            if exit != 0:
-                self.ts.notify(
-                    'Skipping "%s" because "%s %s" failed' %
-                    (scenario.name, step.what, step.text))
-                self.skipped_for_assuming += 1
-                break
-        else:
-            for step in normal:
-                exit = self.run_step(
-                    datadir, scenario, step, shell_prelude, True)
-                step_number += 1
-                self.snapshot_datadir(
-                    tempdir, datadir, scenario, step_number, step)
-                if exit != 0:
-                    ok = False
-                    break
-
-            for step in cleanup:
-                exit = self.run_step(
-                    datadir, scenario, step, shell_prelude, True)
-                step_number += 1
-                self.snapshot_datadir(
-                    tempdir, datadir, scenario, step_number, step)
-                if exit != 0:
-                    ok = False
-                    break
-
-        if not self.settings['snapshot']:
-            shutil.rmtree(tempdir)
+        ok = scenario_runner.run_scenario(scenario, datadir, homedir)
 
         self.remember_scenario_timing(time.time() - started)
         return ok
@@ -297,42 +265,16 @@ class YarnRunner(cliapp.Application):
     def homedir(self, datadir):
         return os.path.join(datadir, 'HOME')
 
-    def clean_env(self):
-        '''Return a clean environment for running tests.'''
-
-        whitelisted = [
-            'PATH',
-        ]
-
-        hardcoded = {
-            'TERM': 'dumb',
-            'SHELL': '/bin/sh',
-            'LC_ALL': 'C',
-            'USER': 'tomjon',
-            'USERNAME': 'tomjon',
-            'LOGNAME': 'tomjon',
-        }
-
-        env = {}
-
-        for key in whitelisted:
-            if key in os.environ:
-                env[key] = os.environ[key]
-
-        for key in hardcoded:
-            env[key] = hardcoded[key]
-
+    def parse_env(self):
         for option_arg in self.settings['env']:
             if '=' not in option_arg:
                 raise cliapp.AppException(
                     '--env argument must contain "=" '
                     'to separate environment variable name and value')
             key, value = option_arg.split('=', 1)
-            env[key] = value
+            yield key, value
 
-        return env
-
-    def run_step(self, datadir, scenario, step, shell_prelude, report_error):
+    def pre_step(self, step, **ignored):
         started = time.time()
 
         self.info('Running step "%s %s"' % (step.what, step.text))
@@ -340,19 +282,12 @@ class YarnRunner(cliapp.Application):
         self.ts['step_name'] = '%s %s' % (step.what, step.text)
         self.steps_run += 1
 
-        m = yarnlib.implements_matches_step(step.implementation, step)
-        assert m is not None
-        env = self.clean_env()
-        env['DATADIR'] = datadir
-        env['SRCDIR'] = os.getcwd()
-        env['HOME'] = self.homedir(datadir)
-        for i, match in enumerate(m.groups('')):
-            env['MATCH_%d' % (i+1)] = match
+        return (started,)
 
-        shell_script = '%s\n\n%s\n' % (
-            shell_prelude, step.implementation.shell)
-        exit, stdout, stderr = cliapp.runcmd_unchecked(
-            ['sh', '-xeuc', shell_script], env=env)
+    def post_step(self, scenario, step, step_number, step_env,
+                  exit, stdout, stderr, pre_step_userdata):
+        stopped = time.time()
+        (started,) = pre_step_userdata
 
         logging.debug('Exit code: %d' % exit)
         if stdout:
@@ -363,20 +298,24 @@ class YarnRunner(cliapp.Application):
             logging.debug('Standard error:\n%s' % self.indent(stderr))
         else:
             logging.debug('Standard error: empty')
-
-        if exit != 0 and report_error:
-            self.error(
-                'ERROR: In scenario "%s"\nstep "%s %s" failed,\n'
-                'with exit code %d:\n'
-                'Standard output from shell command:\n%s'
-                'Standard error from shell command:\n%s' %
-                (scenario.name, step.what, step.text, exit,
-                 self.indent(stdout), self.indent(stderr)))
-
+        if exit != 0:
+            if step.what == 'ASSUMING':
+                self.ts.notify(
+                    'Skipping "%s" because "%s %s" failed' %
+                    (scenario.name, step.what, step.text))
+                self.skipped_for_assuming += 1
+            else:
+                self.error(
+                    'ERROR: In scenario "%s"\nstep "%s %s" failed,\n'
+                    'with exit code %d:\n'
+                    'Standard output from shell command:\n%s'
+                    'Standard error from shell command:\n%s' %
+                    (scenario.name, step.what, step.text, exit,
+                     self.indent(stdout), self.indent(stderr)))
         self.remember_step_timing(
-            '%s %s' % (step.what, step.text), time.time() - started)
-
-        return exit
+            '%s %s' % (step.what, step.text), stopped - started)
+        self.snapshot_datadir(self.tempdir, step_env['DATADIR'],
+                              scenario, step_number, step)
 
     def scenario_dir(self, tempdir, scenario):
         return os.path.join(tempdir, self.nice(scenario.name))
diff --git a/yarnlib/__init__.py b/yarnlib/__init__.py
index e17223a..515082e 100644
--- a/yarnlib/__init__.py
+++ b/yarnlib/__init__.py
@@ -25,6 +25,7 @@ from scenario_step_connector import (implements_matches_step,
                                      ScenarioStepConnector,
                                      StepNotImplementedError,
                                      StepMultipleImplementationsError)
+from scenario_runner import ScenarioRunner
 
 
 from shell_libraries import load_shell_libraries
diff --git a/yarnlib/scenario_runner.py b/yarnlib/scenario_runner.py
new file mode 100644
index 0000000..e0cdf62
--- /dev/null
+++ b/yarnlib/scenario_runner.py
@@ -0,0 +1,202 @@
+# Copyright 2014  Lars Wirzenius and Codethink Limited
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import cliapp
+import os
+
+import yarnlib
+
+
+def default_pre_step(scenario, step, step_number, step_env):
+    '''Default callback run before each step in ScenarioRunner.
+
+    Parameters:
+
+    * `scenario`: The yarnlib.Scenario that is about to be run.
+    * `step`: The yarnlib.ScenarioStep that is about to be run.
+    * `step_number`: Per-scenario counter for the step that is about to be run,
+                     starting from 1.
+    * `step_env`: Environment that will be used for this step.
+    * `return`: Any returned values are passed to `post_step_cb`.
+
+    All parameters are passed as keyword-arguments, so the order of
+    parameters is not important, and unused arguments can be ignored by
+    putting **kwargs in the parameter definition.
+
+    '''
+    pass
+
+
+def default_post_step(scenario, step, step_number, step_env,
+                      exit, stdout, stderr, pre_step_userdata):
+    '''Default callback run after each step in ScenarioRunner.
+
+    Parameters:
+
+    * `scenario`: The yarnlib.Scenario that has just been run.
+    * `step`: The yarnlib.ScenarioStep that has just been run.
+    * `step_number`: Per-scenario counter for the step that is about to be run,
+                     starting from 1.
+    * `step_env`: Environment that was used for this step.
+    * `exit`: Return code of the step that was run.
+    * `stdout`: Standard output of the step that was just run.
+    * `stderr`: Standard error of the step that was just run.
+    * `pre_step_userdata`: Return value from `pre_step_cb`.
+
+    All parameters are passed as keyword-arguments, so the order of
+    parameters is not important, and unused arguments can be ignored by
+    putting **kwargs in the parameter definition.
+
+    '''
+    pass
+
+
+# Arguably this could just be the run_scenario method with all the
+# constructor's parameters passed in, but reeks of poor design with that
+# many parameters.
+# However, decoupling the ScenarioRunner from the scenarios allows the
+# same scenarios to be run against different projects that aim to satisfy
+# the same requirements, or multiple yarn suites on the same project,
+# if the project aims to satisfy multiple sets of requirements.
+# While this use is far less likely than having 1 yarn suite per project,
+# it's a useful metric for deciding how to split the arguments between the
+# constructor and the run_scenario method.
+class ScenarioRunner(object):
+    '''Sets up an environment to run scenarios on a Project.
+
+    Parameters:
+
+    * `shell_prelude`: Prefix to all shell scripts run, e.g. shell libraries.
+    * `srcdir`: Path to the root of the source tree.
+                This is available as the $SRCDIR environment variable
+                and is the directory the scenarios are run from.
+    * `extra_env`: dict, or iterable of pairs of variables to add to
+                   the clean environment provided for every command.
+    * `pre_step_cb`: Callback run before each step in a scenario.
+                     See default_pre_step for an explanation of its signature.
+                     Its return value is passed to `post_step_cb`.
+    * `post_step_cb`: Callback run after each step in a scenario.
+                     See default_post_step for an explanation of its signature.
+    * `cmdrunner`: Function used to run step commands in place of
+                   `cliapp.runcmd_unchecked`, intended to be replaceable
+                   by unit tests.
+
+    '''
+
+
+    def __init__(self, shell_prelude, srcdir, extra_env=(),
+                 pre_step_cb=default_pre_step, post_step_cb=default_post_step,
+                 cmdrunner=cliapp.runcmd_unchecked):
+        self.shell_prelude = shell_prelude
+        self.srcdir = srcdir
+        self.env = self.clean_env(extra_env, SRCDIR=srcdir)
+        self.pre_step_cb = pre_step_cb
+        self.post_step_cb = post_step_cb
+        self.cmdrunner = cmdrunner
+
+    def run_scenario(self, scenario, datadir, homedir):
+        assuming = [s for s in scenario.steps if s.what == 'ASSUMING']
+        cleanup = [s for s in scenario.steps if s.what == 'FINALLY']
+        normal = [s for s in scenario.steps if s not in assuming + cleanup]
+
+        ok = True
+        step_number = 1
+
+        scenario_env = dict(self.env)
+        scenario_env['HOME'] = homedir
+        scenario_env['DATADIR'] = datadir
+
+        for step in assuming:
+            exit = self.run_step(scenario, step, scenario_env, step_number)
+            step_number += 1
+            if exit != 0:
+                break
+        else:
+            for step in normal:
+                exit = self.run_step(scenario, step, scenario_env, step_number)
+                step_number += 1
+                if exit != 0:
+                    ok = False
+                    break
+
+            for step in cleanup:
+                exit = self.run_step(scenario, step, scenario_env, step_number)
+                step_number += 1
+                if exit != 0:
+                    ok = False
+                    break
+
+        return ok
+
+    def run_step(self, scenario, step, scenario_env, step_number):
+        m = yarnlib.implements_matches_step(step.implementation, step)
+        assert m is not None
+        step_env = dict(scenario_env)
+        for i, match in enumerate(m.groups('')):
+            step_env['MATCH_%d' % (i+1)] = match
+
+        # All parameters passed as keyword-arguments, so that the callback
+        # may declare parameters in any order, and ignore any parameters
+        # by specifying **kwargs
+        pre_step_userdata = self.pre_step_cb(scenario=scenario, step=step,
+                                             step_number=step_number,
+                                             step_env=step_env)
+
+        shell_script = '%s\n\n%s\n' % (
+            self.shell_prelude, step.implementation.shell)
+        exit, stdout, stderr = self.cmdrunner(
+            ['sh', '-xeuc', shell_script], env=step_env, cwd=self.srcdir)
+
+        # All parameters passed as keyword-arguments, so that the callback
+        # may declare parameters in any order, and ignore any parameters
+        # by specifying **kwargs
+        self.post_step_cb(scenario=scenario, step=step,
+                          step_number=step_number, step_env=step_env,
+                          exit=exit, stdout=stdout, stderr=stderr,
+                          pre_step_userdata=pre_step_userdata)
+
+        return exit
+
+    @staticmethod
+    def clean_env(extra_env, **kwarg_env):
+        '''Return a clean environment for running tests.'''
+
+        whitelisted = [
+            'PATH',
+        ]
+
+        hardcoded = {
+            'TERM': 'dumb',
+            'SHELL': '/bin/sh',
+            'LC_ALL': 'C',
+            'USER': 'tomjon',
+            'USERNAME': 'tomjon',
+            'LOGNAME': 'tomjon',
+        }
+
+        env = {}
+
+        for key in whitelisted:
+            if key in os.environ:
+                env[key] = os.environ[key]
+
+        env.update(hardcoded)
+        env.update(extra_env)
+        env.update(kwarg_env)
+
+        return env
diff --git a/yarnlib/scenario_runner_tests.py b/yarnlib/scenario_runner_tests.py
new file mode 100644
index 0000000..cda6397
--- /dev/null
+++ b/yarnlib/scenario_runner_tests.py
@@ -0,0 +1,129 @@
+# Copyright 2014  Codethink Limited
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import itertools
+import unittest
+
+import yarnlib
+
+
+class ScenarioRunnerEnvironmentTests(unittest.TestCase):
+
+    def setUp(self):
+        self.cmdlog = []
+        self.scenario = yarnlib.Scenario('foo')
+        step = yarnlib.ScenarioStep('THEN', 'foo bar')
+        step.implementation = yarnlib.Implementation('THEN', r'foo (\S+)',
+                                                     'echo foo $MATCH_1')
+        self.scenario.steps = [step]
+
+    def fake_cmdrunner(self, cmd, env=None, cwd=None):
+        self.cmdlog.append((cmd, env, cwd))
+        return 0, '', ''
+
+    def test_command_env(self):
+        sr = yarnlib.ScenarioRunner('', '/tmp', {'FOO': 'bar'},
+                                    cmdrunner=self.fake_cmdrunner)
+        sr.run_scenario(self.scenario, '/tmp/datadir', '/tmp/home')
+        cmd, cmd_env, cmd_cwd = self.cmdlog[0]
+        self.assertIn('FOO', cmd_env)
+        self.assertEqual(cmd_env['FOO'], 'bar')
+        self.assertEqual(cmd_env['SRCDIR'], '/tmp')
+        self.assertEqual(cmd_env['DATADIR'], '/tmp/datadir')
+        self.assertEqual(cmd_env['HOME'], '/tmp/home')
+        self.assertEqual(cmd_env['MATCH_1'], 'bar')
+        self.assertEqual('/tmp', cmd_cwd)
+
+    def test_command_args(self):
+        prelude = 'TESTVAR=foo\n'
+        impl_shell = self.scenario.steps[0].implementation.shell
+        sr = yarnlib.ScenarioRunner(prelude, '/tmp',
+                                    cmdrunner=self.fake_cmdrunner)
+        sr.run_scenario(self.scenario, '/tmp/datadir', '/tmp/home')
+        shell, opts, command = self.cmdlog[0][0]
+        self.assertTrue(shell.endswith('sh'))
+        self.assertTrue(command.startswith(prelude))
+        self.assertIn(impl_shell, command)
+
+
+class ScenarioRunnerFlowTests(unittest.TestCase):
+
+    def setUp(self):
+        self.cmdlog = []
+        self.scenario_runner = (
+            yarnlib.ScenarioRunner('', 'srcdir',
+                                   cmdrunner=self.fake_cmdrunner))
+        for verb, cmd in itertools.product(('ASSUMING', 'THEN', 'FINALLY'),
+                                           ('true', 'false')):
+            step = yarnlib.ScenarioStep(verb, cmd)
+            step.implementation = yarnlib.Implementation(verb, cmd, cmd)
+            setattr(self, '%s_%s' % (verb.lower(), cmd), step)
+
+    def fake_cmdrunner(self, cmd, *args, **kwargs):
+        self.cmdlog.append(cmd)
+        if 'false' in cmd[-1]:
+            return 1, '', ''
+        return 0, '', ''
+
+    def run_scenario(self, *steps):
+        scenario = yarnlib.Scenario('foo')
+        scenario.steps = steps
+        return self.scenario_runner.run_scenario(scenario, 'data', 'home')
+
+    def test_assuming_skips_remaining(self):
+        ok = self.run_scenario(
+            self.assuming_false,
+            self.then_true,
+            self.finally_true,
+        )
+        self.assertTrue(ok)
+        self.assertEqual(len(self.cmdlog), 1)
+
+    def test_cleanup_run_on_success(self):
+        ok = self.run_scenario(
+            self.assuming_true,
+            self.then_true,
+            self.finally_true,
+        )
+        self.assertTrue(ok)
+        self.assertEqual(len(self.cmdlog), 3)
+
+    def test_cleanup_run_on_failure(self):
+        ok = self.run_scenario(
+            self.then_false,
+            self.finally_true,
+        )
+        self.assertFalse(ok)
+        self.assertEqual(len(self.cmdlog), 2)
+
+    def test_skip_steps_after_failure(self):
+        ok = self.run_scenario(
+            self.then_false,
+            self.then_true,
+        )
+        self.assertFalse(ok)
+        self.assertEqual(len(self.cmdlog), 1)
+
+    def test_cleanup_failure(self):
+        ok = self.run_scenario(
+            self.then_true,
+            self.finally_false,
+            self.finally_true,
+        )
+        self.assertFalse(ok)
+        self.assertEqual(len(self.cmdlog), 2)