From 49846bf875da7b7b44ee8fc8744bd84b6e65cf9b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 18 Oct 2013 17:51:47 +0100 Subject: Report number of scenarios skipped due to ASSUMING --- NEWS | 6 ++++++ yarn | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/NEWS b/NEWS index 47fbd88..f794e69 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,12 @@ NEWS for cmdtest This file summarizes changes between releases of cmdtest. +Version 0.11, released UNRELEASED +--------------------------------- + +* Report number of scenarios skipped due to an ASSUMING step + failing. + Version 0.10, released 2013-10-05 --------------------------------- diff --git a/yarn b/yarn index f4abcdc..f8ca0cd 100755 --- a/yarn +++ b/yarn @@ -125,6 +125,7 @@ class YarnRunner(cliapp.Application): self.info('Found %d scenarios' % len(scenarios)) self.scenarios_run = 0 + self.skipped_for_assuming = 0 self.steps_run = 0 self.timings = [] @@ -149,6 +150,9 @@ class YarnRunner(cliapp.Application): '(%d total steps), ' 'in %.1f seconds' % (self.scenarios_run, self.steps_run, duration)) + if self.skipped_for_assuming: + print ('Scenarios SKIPPED due to ASSUMING step failing: %d' + % self.skipped_for_assuming) if self.settings['timings']: self.report_timings() @@ -299,6 +303,7 @@ class YarnRunner(cliapp.Application): self.ts.notify( 'Skipping "%s" because "%s %s" failed' % (scenario.name, step.what, step.text)) + self.skipped_for_assuming += 1 break else: for step in normal: -- cgit v1.2.1 From e16eba47205f91271b77dc71f960427da5c76f72 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 20 Oct 2013 10:13:27 +0100 Subject: Fix cmdtest --test --- NEWS | 7 +++++++ cmdtest | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/NEWS b/NEWS index f794e69..2f49678 100644 --- a/NEWS +++ b/NEWS @@ -6,9 +6,16 @@ This file summarizes changes between releases of cmdtest. Version 0.11, released UNRELEASED --------------------------------- +For yarn: + * Report number of scenarios skipped due to an ASSUMING step failing. +For cmdtest: + +* The `--test` (`-t`) option should now work again. Thank you + to Kalle Valo for prodding me. + Version 0.10, released 2013-10-05 --------------------------------- diff --git a/cmdtest b/cmdtest index b8c6efa..d7f329e 100755 --- a/cmdtest +++ b/cmdtest @@ -120,6 +120,13 @@ class CommandTester(cliapp.Application): for test in self.settings['test']: matches = glob.glob(os.path.join(dirname, test + '.*')) filenames += [os.path.basename(x) for x in matches] + + basenames = ['setup_once', 'setup', 'teardown', 'teardown_once'] + for basename in basenames: + filename = os.path.join(dirname, basename) + if os.path.exists(filename): + filenames.append(basename) + return filenames def setup_tempdir(self): -- cgit v1.2.1 From 0aad99d630ed073c7d071906f81c8aeba8196dc2 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 24 Oct 2013 20:42:26 +0100 Subject: Start a yarn manual file --- yarn-doc/index.mdwn | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 yarn-doc/index.mdwn diff --git a/yarn-doc/index.mdwn b/yarn-doc/index.mdwn new file mode 100644 index 0000000..7deaffb --- /dev/null +++ b/yarn-doc/index.mdwn @@ -0,0 +1,57 @@ +% Yarn manual + +Yarn +==== + +FIXME: This will become a manual for writing test suites in yarn. It +is currently not yet written. + +Outline +------- + +* Introduction + - what is yarn? + - who is yarn for? + - who are the test suites written in yarn for? + - what kinds of testing is yarn for? + - why yarn instead of other tools? + - why not cmdtest? + - NOT installation instructions +* Examples + - a test suite for "hello world" + - make the files available so people can try things for themselves + - a few simple scenarios +* The yarn testing language + - Markdown with blockquotes for the executable code + - SCENARIO + the step-wise keywords + - IMPLEMENTS sections +* Running yarn + - command line syntax + - examples of various ways to run yarn in different scenarios: + - how to run just one scenario + - how to run yarn under cron or jenkins + - formatting a test suite in yarn with pandoc +* Best practices + - this chapter will describe best practices for writing test suites + with yarn + - how to structure the files: what to put in each *.yarn file, e.g., + where should IMPLEMENTS go + - how to write test suites that make it easy to debug things when a + test case fails + - good phrasing guidelines for yarn scenario names and step names + - what things are good to keep visible to the reader, what are + better hidden inside impementations of steps, with examples from + real projects using yarn + - guidelines for well-defined steps that are easy to understand and + easy to implement + - anti-patterns: things that are good to avoid + - make tests fast + - make test code be obviously correct; make test code be the best + code + - when is it OK to skip scenarios? +* Case studies + - this chapter will discuss ways to use yarn in things that are not + just "run this program and examine the output" + - start a daemon in the background, kill it at the end of a scenario + - how to use a really heavy-weight thing in test suites (e.g., start + a database server for all scenarios to share) -- cgit v1.2.1 From e67bf025e4a521352147c29d0175f31469134407 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Sun, 27 Oct 2013 08:44:04 +0000 Subject: First contributions add 'Mission' section - wording to be agreed changed by Lars add 'Document Status' section - 'Done', 'New' and 'Coming soon' --- yarn-doc/index.mdwn | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/yarn-doc/index.mdwn b/yarn-doc/index.mdwn index 7deaffb..620572d 100644 --- a/yarn-doc/index.mdwn +++ b/yarn-doc/index.mdwn @@ -6,6 +6,44 @@ Yarn FIXME: This will become a manual for writing test suites in yarn. It is currently not yet written. +Mission +------- +This manual will provide all the information needed by Yarn users to enable +them to use Yarn effectively in their development projects. + +The information will be + +* easy to find +* easy to navigate +* easy to use + +The information will include details of + +* how to perform certain tasks +* why things are done in particular ways + +Document Status +--------------- + +What's Done +=========== + +* Outline + +What's New +========== + +* Mission +* this Document Status section + +What's Next +=========== + +* Test language specification (from README.yarn) +* `yarn`'s command line +* How to embed `yarn` in Markdown + + Outline ------- -- cgit v1.2.1 From c7fbccddef6a262480a5f8790d5b330a4eeb6b65 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Wed, 30 Oct 2013 19:15:35 +0000 Subject: S9399 Yarn manual - Test language specification (from README.yarn) Also started Introduction - wrote a skeleton and populated the 'What is yarn?' section Moved some of the introductory text into 'Writing Scenarios' chapter, and made the Language Specification a sub-section of that. Tweaked heading and indentations --- yarn-doc/index.mdwn | 207 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 198 insertions(+), 9 deletions(-) diff --git a/yarn-doc/index.mdwn b/yarn-doc/index.mdwn index 620572d..a6a3f02 100644 --- a/yarn-doc/index.mdwn +++ b/yarn-doc/index.mdwn @@ -25,24 +25,213 @@ The information will include details of Document Status --------------- -What's Done -=========== +### What's Done * Outline - -What's New -========== - * Mission * this Document Status section -What's Next -=========== +### What's New + +* Writing Scenarios + * Test Language Specification +* Introduction + * Skeleton + * What is `yarn`? -* Test language specification (from README.yarn) + +### What's Next + +* Introduction + * Remaining sections * `yarn`'s command line * How to embed `yarn` in Markdown +Introduction +------------ + +### What is `yarn`? + +`yarn` is a scenario testing tool: you write a scenario describing how a +user uses your software and what should happen, and express, using +very lightweight syntax, the scenario in such a way that it can be tested +automatically. The scenario has a simple, but strict structure: + + SCENARIO name of scenario + GIVEN some setup for the test + WHEN thing that is to be tested happens + THEN the post-conditions must be true + +As an example, consider a very short test scenario for verifying that +a backup program works, at least for one simple case. + + SCENARIO basic backup and restore + GIVEN some live data in a directory + AND an empty backup repository + WHEN a backup is made + THEN the data can be restored + +(Note the addition of AND: you can have multiple GIVEN, WHEN, and +THEN statements. The AND keyword makes the text be more readable.) + + +### Who is `yarn` for? + +### Who are the test suites written in `yarn` for? + +### What kinds of testing is `yarn` for? + +### Why `yarn` instead of other tools? + +### Why not cmdtest? + +Writing Scenarios +----------------- + +Scenarios are meant to be written in mostly human readable language. +However, they are not free form text. In addition to the GIVEN/WHEN/THEN +structure, the text for each of the steps needs a computer-executable +implementation. This is done by using IMPLEMENTS. The backup scenario +from above might be implemented as follows: + + IMPLEMENTS GIVEN some live data in a directory + rm -rf "$DATADIR/data" + mkdir "$DATADIR/data" + echo foo > "$DATADIR/data/foo" + + IMPLEMENTS GIVEN an empty backup repository + rm -rf "$DATADIR/repo" + mkdir "$DATADIR/repo" + + IMPLEMENTS WHEN a backup is made + backup-program -r "$DATADIR/repo" "$DATADIR/data" + + IMPLEMENTS THEN the data can be restored + mkdir "$DATADIR/restored" + restore-program -r "$DATADIR/repo" "$DATADIR/restored" + diff -rq "$DATADIR/data" "$DATADIR/restored" + +Each "IMPLEMENT GIVEN" (or WHEN, THEN) is followed by a regular +expression on the same line, and then a shell script that gets executed +to implement any step that matches the regular expression. The +implementation can extract data from the match as well: for example, +the regular expression might allow a file size to be specified. + +The above example seems a bit silly, of course: why go to the effort +to obfuscate the various steps? The answer is that the various steps, +implemented using IMPLEMENTS, can be combined in many ways, to test +different aspects of the program being tested. In effect, the IMPLEMENTS +sections provide a vocabulary which the scenario writer can use to +express a variety of usefully different scenarios, which together +test all the aspects of the software that need to be tested. + +Moreover, by making the step descriptions be human language +text, matched by regular expressions, most of the test can +hopefully be written, and understood, by non-programmers. Someone +who understands what a program should do, could write tests +to verify its behaviour. The implementations of the various +steps need to be implemented by a programmer, but given a +well-designed set of steps, with enough flexibility in their +implementation, that quite a good test suite can be written. + +### Test Language Specification + +A test document is written in [Markdown][markdown], with block +quoted code blocks being interpreted specially. Each block +must follow the syntax defined here. + +* Every step in a scenario is one line, and starts with a keyword. + +* Each implementation (IMPLEMENTS) starts as a new block, and + continues until there is a block that starts with another + keyword. + +The following keywords are defined. + +* **SCENARIO** starts a new scenario. The rest of the line is the name of + the scenario. The name is used for documentation and reporting + purposes only and has no semantic meaning. SCENARIO MUST be the + first keyword in a scenario, with the exception of IMPLEMENTS. + The set of documents passed in a test run may define any number of + scenarios between them, but there must be at least one or it is a + test failure. The IMPLEMENTS sections are shared between the + documents and scenarios. + +* **ASSUMING** defines a condition for the scenario. The rest of the + line is "matched text", which gets implemented by an + IMPLEMENTS section. If the code executed by the implementation + fails, the scenario is skipped. + +* **GIVEN** prepares the world for the test to run. If + the implementation fails, the scenario fails. + +* **WHEN** makes the change to the world that is to be tested. + If the code fails, the scenario fails. + +* **THEN** verifies that the changes made by the GIVEN steps + did the right thing. If the code fails, the scenario fails. + +* **FINALLY** specifies how to clean up after a scenario. If the code + fails, the scenario fails. All FINALLY blocks get run either when + encountered in the scenario flow, or at the end of the scenario, + regardless of whether the scenario is failing or not. + +* **AND** acts as ASSUMING, GIVEN, WHEN, THEN, or FINALLY: whichever + was used last. It must not be used unless the previous step was + one of those, or another AND. + +* **IMPLEMENTS** is followed by one of ASSUMING, GIVEN, WHEN, or THEN, + and a PCRE regular expression, all on one line, and then further + lines of shell commands until the end of the block quoted code + block. Markdown is unclear whether an empty line (no characters, + not even whitespace) between two block quoted code blocks starts a + new one or not, so we resolve the ambiguity by specifiying that a + code block directly following a code block is a continuation unless + it starts with one of the scenario testing keywords. + + The shell commands get parenthesised parts of the match of the + regular expression as environment variables (`$MATCH_1` etc). For + example, if the regexp is "a (\d+) byte file", then `$MATCH_1` gets + set to the number matched by `\d+`. + + The test runner creates a temporary directory, whose name is + given to the shell code in the `DATADIR` environment variable. + + The test runner sets the `SRCDIR` environment variable to the + path to the directory it was invoked from (by convention, the + root of the source tree of the project). + + The test runner removes all other environment variables, except + `TERM`, `USER`, `USERNAME`, `LOGNAME`, `HOME`, and `PATH`. It also + forces `SHELL` set to `/bin/sh`, and `LC_ALL` set to `C`, in order + to have as clean an environment as possible for tests to run in. + + The shell commands get invoked with `/bin/sh -eu`, and need to + be written accordingly. Be careful about commands that return a + non-zero exit code. There will eventually be a library of shell + functions supplied which allow handling the testing of non-zero + exit codes cleanly. In addition functions for handling stdout and + stderr will be provided. + + The code block of an IMPLEMENTS block fails if the shell + invocation exits with a non-zero exit code. Output to stderr is + not an indication of failure. Any output to stdout or stderr may + or may not be shown to the user. + +Semantics: + +* The name of each scenario (given with SCENARIO) must be unique. +* All names of scenarios and steps will be normalised before use + (whitespace collapse, leading and trailing whitespace +* Every ASSUMING, GIVEN, WHEN, THEN, FINALLY must be matched by + exactly one IMPLEMENTS. The test runner checks this before running + any code. +* Every IMPLEMENTS may match any number of ASSUMING, GIVEN, WHEN, + THEN, or FINALLY. The test runner may warn if an IMPLEMENTS is unused. +* If ASSUMING fails, that scenario is skipped, and any FINALLY steps + are not run. + + Outline ------- -- cgit v1.2.1 From 4f1b08500e164a3a555a508260adbfa4078a965d Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Wed, 30 Oct 2013 19:46:54 +0000 Subject: Fix nested bullets --- yarn-doc/index.mdwn | 80 ++++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/yarn-doc/index.mdwn b/yarn-doc/index.mdwn index a6a3f02..d63dee0 100644 --- a/yarn-doc/index.mdwn +++ b/yarn-doc/index.mdwn @@ -237,48 +237,48 @@ Outline ------- * Introduction - - what is yarn? - - who is yarn for? - - who are the test suites written in yarn for? - - what kinds of testing is yarn for? - - why yarn instead of other tools? - - why not cmdtest? - - NOT installation instructions + - what is yarn? + - who is yarn for? + - who are the test suites written in yarn for? + - what kinds of testing is yarn for? + - why yarn instead of other tools? + - why not cmdtest? + - NOT installation instructions * Examples - - a test suite for "hello world" - - make the files available so people can try things for themselves - - a few simple scenarios + - a test suite for "hello world" + - make the files available so people can try things for themselves + - a few simple scenarios * The yarn testing language - - Markdown with blockquotes for the executable code - - SCENARIO + the step-wise keywords - - IMPLEMENTS sections + - Markdown with blockquotes for the executable code + - SCENARIO + the step-wise keywords + - IMPLEMENTS sections * Running yarn - - command line syntax - - examples of various ways to run yarn in different scenarios: - - how to run just one scenario - - how to run yarn under cron or jenkins - - formatting a test suite in yarn with pandoc + - command line syntax + - examples of various ways to run yarn in different scenarios: + - how to run just one scenario + - how to run yarn under cron or jenkins + - formatting a test suite in yarn with pandoc * Best practices - - this chapter will describe best practices for writing test suites - with yarn - - how to structure the files: what to put in each *.yarn file, e.g., - where should IMPLEMENTS go - - how to write test suites that make it easy to debug things when a - test case fails - - good phrasing guidelines for yarn scenario names and step names - - what things are good to keep visible to the reader, what are - better hidden inside impementations of steps, with examples from - real projects using yarn - - guidelines for well-defined steps that are easy to understand and - easy to implement - - anti-patterns: things that are good to avoid - - make tests fast - - make test code be obviously correct; make test code be the best - code - - when is it OK to skip scenarios? + - this chapter will describe best practices for writing test suites + with yarn + - how to structure the files: what to put in each *.yarn file, e.g., + where should IMPLEMENTS go + - how to write test suites that make it easy to debug things when a + test case fails + - good phrasing guidelines for yarn scenario names and step names + - what things are good to keep visible to the reader, what are + better hidden inside impementations of steps, with examples from + real projects using yarn + - guidelines for well-defined steps that are easy to understand and + easy to implement + - anti-patterns: things that are good to avoid + - make tests fast + - make test code be obviously correct; make test code be the best + code + - when is it OK to skip scenarios? * Case studies - - this chapter will discuss ways to use yarn in things that are not - just "run this program and examine the output" - - start a daemon in the background, kill it at the end of a scenario - - how to use a really heavy-weight thing in test suites (e.g., start - a database server for all scenarios to share) + - this chapter will discuss ways to use yarn in things that are not + just "run this program and examine the output" + - start a daemon in the background, kill it at the end of a scenario + - how to use a really heavy-weight thing in test suites (e.g., start + a database server for all scenarios to share) -- cgit v1.2.1 From c4cbaf65ea8dff900b98b1a7a7f3d71cb0b19c41 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 11 Nov 2013 18:14:01 +0000 Subject: Given one invalid yarn and one valid one, check yarn reports the correct one as invalid --- yarn.tests/two-no-thens.exit | 1 + yarn.tests/two-no-thens.script | 23 +++++++++++++++++++++++ yarn.tests/two-no-thens.stderr | 3 +++ 3 files changed, 27 insertions(+) create mode 100755 yarn.tests/two-no-thens.exit create mode 100755 yarn.tests/two-no-thens.script create mode 100755 yarn.tests/two-no-thens.stderr diff --git a/yarn.tests/two-no-thens.exit b/yarn.tests/two-no-thens.exit new file mode 100755 index 0000000..d00491f --- /dev/null +++ b/yarn.tests/two-no-thens.exit @@ -0,0 +1 @@ +1 diff --git a/yarn.tests/two-no-thens.script b/yarn.tests/two-no-thens.script new file mode 100755 index 0000000..a6a67fd --- /dev/null +++ b/yarn.tests/two-no-thens.script @@ -0,0 +1,23 @@ +#!/bin/sh + +set -eu + + +cat < "$DATADIR/1.yarn" + SCENARIO valid scenario + GIVEN statement 1 + THEN statement 2 + + SCENARIO invalid scenario + GIVEN statement 1 + + IMPLEMENTS GIVEN statement (\d) + true + + IMPLEMENTS THEN statement (\d) + true + + +EOF + +./run-yarn "$DATADIR/1.yarn" diff --git a/yarn.tests/two-no-thens.stderr b/yarn.tests/two-no-thens.stderr new file mode 100755 index 0000000..4599b30 --- /dev/null +++ b/yarn.tests/two-no-thens.stderr @@ -0,0 +1,3 @@ +ERROR: Some scenarios have no THENs: + "invalid scenario" + -- cgit v1.2.1 From c6c8f2c96eb90f1af6ff9115ce0136eb82389b5b Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 11 Nov 2013 18:22:56 +0000 Subject: Fix bug where yarn reports all yarns as having no THENS --- yarn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn b/yarn index f8ca0cd..c3cba28 100755 --- a/yarn +++ b/yarn @@ -198,7 +198,7 @@ class YarnRunner(cliapp.Application): if no_thens: raise cliapp.AppException( 'Some scenarios have no THENs:\n%s' % - ''.join(' "%s"\n' % s.name for s in scenarios)) + ''.join(' "%s"\n' % s.name for s in no_thens)) def connect_implementations(self, scenarios, implementations): for scenario in scenarios: -- cgit v1.2.1 From b64f4810bbe24c0384535afb202d6c2bd2ee6214 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 15 Nov 2013 14:25:45 +0000 Subject: Add installation instructions to README.yarn Suggested-by: David Harrison --- README.yarn | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.yarn b/README.yarn index 16f7522..5f1c7a2 100644 --- a/README.yarn +++ b/README.yarn @@ -1,6 +1,26 @@ README for yarn, a scenario testing tool ======================================== +Installation +------------ + +* You need Python 2. Yarn's dependencies do not work with Python 3. + (Yet.) +* Install ttystatus: +* Install cliapp: +* Install Python-Markdown: + +* You can install them from source, or using your operating system's + package manager (if they've been packaged for your operating system; + Debian has them: `python-ttystatus`, `python-cliapp`, + `python-markdown`). +* Similarly, install yarn from source or with the package manager + (in Debian: `cmdtest`). +* Installation from source requires cloning the git repository and + running `python setup.py install`, perhaps with additional options + (see `--help`) to specify installation location, etc. Read also the + installation documentation of the libraries. + Introduction ------------ -- cgit v1.2.1 From 9800dff2f907cd4b5d2bf70bf66eca42aee7d658 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 15 Nov 2013 14:35:36 +0000 Subject: Update NEWS about no-thens bug --- NEWS | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 2f49678..4b8bb58 100644 --- a/NEWS +++ b/NEWS @@ -8,8 +8,10 @@ Version 0.11, released UNRELEASED For yarn: -* Report number of scenarios skipped due to an ASSUMING step - failing. +* Report number of scenarios skipped due to an ASSUMING step failing. + * Fix the error message for reporting scenarios without THEN steps + to include the names of those scenarios only, rather than all + scenarios. Patch by Pete Fotheringham. For cmdtest: -- cgit v1.2.1 From 099c7d84298c12db09979903720fa44c615e1f7b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 21 Jan 2014 09:55:24 +0000 Subject: Make step progress reporting nicer A count of all steps is nicer than separate scenario/step counts. --- yarn | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/yarn b/yarn index c3cba28..24e982b 100755 --- a/yarn +++ b/yarn @@ -109,9 +109,9 @@ class YarnRunner(cliapp.Application): self.ts = ttystatus.TerminalStatus(period=0.001) if not self.settings['quiet'] and not self.settings['verbose']: self.ts.format( - '%ElapsedTime() %Index(scenario,scenarios): ' + '%ElapsedTime() %Index(current_step,all_steps): ' '%String(scenario_name): ' - 'step %Index(step,steps): %String(step_name)') + '%String(step_name)') scenarios, implementations = self.parse_scenarios(args) self.check_there_are_scenarios(scenarios) @@ -120,10 +120,13 @@ class YarnRunner(cliapp.Application): self.connect_implementations(scenarios, implementations) shell_prelude = self.load_shell_libraries() - self.ts['scenarios'] = scenarios - self.ts['num_scenarios'] = len(scenarios) self.info('Found %d scenarios' % len(scenarios)) + all_steps = [] + for scenario in scenarios: + all_steps.extend(scenario.steps) + self.ts['all_steps'] = all_steps + self.scenarios_run = 0 self.skipped_for_assuming = 0 self.steps_run = 0 @@ -265,13 +268,13 @@ class YarnRunner(cliapp.Application): started = time.time() self.info('Running scenario %s' % scenario.name) - self.ts['scenario'] = scenario self.ts['scenario_name'] = scenario.name - self.ts['steps'] = scenario.steps self.scenarios_run += 1 if self.settings['no-act']: self.info('Pretending everything went OK') + for step in scenario.steps: + self.ts['current_step'] = step self.remember_scenario_timing(time.time() - started) return True @@ -372,7 +375,7 @@ class YarnRunner(cliapp.Application): started = time.time() self.info('Running step "%s %s"' % (step.what, step.text)) - self.ts['step'] = step + self.ts['current_step'] = step self.ts['step_name'] = '%s %s' % (step.what, step.text) self.steps_run += 1 -- cgit v1.2.1 From 058d1ae7730d8306242035347cacd1ca7b9787d7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 21 Jan 2014 20:50:34 +0000 Subject: Make yarn set $HOME to an existing directory --- NEWS | 5 +++++ yarn | 8 +++++++- yarn.tests/env.script | 5 +++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 4b8bb58..555c610 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,11 @@ For yarn: to include the names of those scenarios only, rather than all scenarios. Patch by Pete Fotheringham. +* Yarn now sets `$HOME` to a directory in `$DATADIR`, and creates + that directory. This means test suites can assume `$HOME` exists, + but don't use the user's real home directory, which is important for + test environment hygiene. + For cmdtest: * The `--test` (`-t`) option should now work again. Thank you diff --git a/yarn b/yarn index 24e982b..ef19b94 100755 --- a/yarn +++ b/yarn @@ -289,6 +289,9 @@ class YarnRunner(cliapp.Application): datadir = self.datadir(tempdir, scenario) os.mkdir(datadir) self.info('DATADIR is %s' % datadir) + homedir = self.homedir(datadir) + os.mkdir(homedir) + self.info('HOME for tests is %s' % homedir) assuming = [s for s in scenario.steps if s.what == 'ASSUMING'] cleanup = [s for s in scenario.steps if s.what == 'FINALLY'] @@ -335,6 +338,9 @@ class YarnRunner(cliapp.Application): self.remember_scenario_timing(time.time() - started) return ok + def homedir(self, datadir): + return os.path.join(datadir, 'HOME') + def clean_env(self): '''Return a clean environment for running tests.''' @@ -349,7 +355,6 @@ class YarnRunner(cliapp.Application): 'USER': 'tomjon', 'USERNAME': 'tomjon', 'LOGNAME': 'tomjon', - 'HOME': '/this/path/does/not/exist', } env = {} @@ -384,6 +389,7 @@ class YarnRunner(cliapp.Application): env = self.clean_env() env['DATADIR'] = datadir env['SRCDIR'] = os.getcwd() + env['HOME'] = self.homedir(datadir) for i, match in enumerate(m.groups('')): env['MATCH_%d' % (i+1)] = match diff --git a/yarn.tests/env.script b/yarn.tests/env.script index d14a21a..d24172d 100755 --- a/yarn.tests/env.script +++ b/yarn.tests/env.script @@ -7,6 +7,7 @@ cat << 'EOF' > "$DATADIR/env.yarn" THEN DATADIR is set AND SRCDIR is set AND NOTSET is not set + AND HOME is prefixed by DATADIR IMPLEMENTS THEN (\S+) is set env @@ -14,6 +15,10 @@ cat << 'EOF' > "$DATADIR/env.yarn" IMPLEMENTS THEN (\S+) is not set ! env | grep "^$MATCH_1=" + + IMPLEMENTS THEN (\S+) is prefixed by (\S+) + x=$(env | sed -n "/^$MATCH_2=/s///p") + env | grep "^$MATCH_1=$x/" EOF NOTSET=foo ./run-yarn "$DATADIR/env.yarn" -- cgit v1.2.1 From 0da0eb42be3ec77416d154336e36a2c6c19b1c6b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 21 Jan 2014 21:01:53 +0000 Subject: Add --allow-missing-steps --- NEWS | 4 ++++ yarn | 22 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 555c610..5c99718 100644 --- a/NEWS +++ b/NEWS @@ -18,6 +18,10 @@ For yarn: but don't use the user's real home directory, which is important for test environment hygiene. +* New option `--allow-missing-steps` to allow running a test suite + with some steps missing. The scenarios with missing steps will be + skipped. Suggested by Pete Fotheringham. + For cmdtest: * The `--test` (`-t`) option should now work again. Thank you diff --git a/yarn b/yarn index ef19b94..af25d21 100755 --- a/yarn +++ b/yarn @@ -79,6 +79,11 @@ class YarnRunner(cliapp.Application): ['timings'], 'report wall clock time for each scenario and step') + self.settings.boolean( + ['allow-missing-steps'], + 'allow scenarios to reference steps that do not exist, ' + 'by warning about them, but otherwise ignoring the scenarios') + def info(self, msg): if self.settings['verbose']: logging.info(msg) @@ -117,7 +122,7 @@ class YarnRunner(cliapp.Application): self.check_there_are_scenarios(scenarios) self.check_for_duplicate_scenario_names(scenarios) self.check_for_thens(scenarios) - self.connect_implementations(scenarios, implementations) + scenarios = self.connect_implementations(scenarios, implementations) shell_prelude = self.load_shell_libraries() self.info('Found %d scenarios' % len(scenarios)) @@ -204,9 +209,17 @@ class YarnRunner(cliapp.Application): ''.join(' "%s"\n' % s.name for s in no_thens)) def connect_implementations(self, scenarios, implementations): + new_list = [] for scenario in scenarios: + missing_step = False for step in scenario.steps: - self.connect_implementation(scenario, step, implementations) + self.connect_implementation( + scenario, step, implementations) + if step.implementation is None: + missing_step = True + if not missing_step: + new_list.append(scenario) + return new_list def connect_implementation(self, scenario, step, implementations): matching = [i for i in implementations @@ -214,6 +227,11 @@ class YarnRunner(cliapp.Application): self.implements_matches_step(i, step)] if len(matching) == 0: + if self.settings['allow-missing-steps']: + self.warning( + 'Scenario %s has missing step %s %s' % + (scenario.name, step.what, step.text)) + return raise cliapp.AppException( 'Scenario "%s", step "%s %s" has no matching ' 'implementation' % -- cgit v1.2.1 From ea926a07a4b800bcd87d0841c213ad02faae00f3 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 5 Feb 2014 16:20:49 +0000 Subject: elements: remove unused execute method --- yarnlib/elements.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/yarnlib/elements.py b/yarnlib/elements.py index 9eeb1b7..f6b6f76 100644 --- a/yarnlib/elements.py +++ b/yarnlib/elements.py @@ -1,4 +1,4 @@ -# Copyright 2013 Lars Wirzenius +# Copyright 2013-2014 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,8 +44,3 @@ class Implementation(object): self.regexp = regexp self.shell = shell - def execute(self): - exit, out, err = cliapp.runcmd_unchecked( - ['sh', '-c', 'set -eu\n' + self.shell]) - return exit - -- cgit v1.2.1 From d80685642ad8d01da34e21f3957df0c06075248a Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 5 Feb 2014 14:13:52 +0000 Subject: Factor scenario validation into yarnlib --- yarn | 37 ++--------------- yarnlib/__init__.py | 4 +- yarnlib/scenario_validator.py | 82 +++++++++++++++++++++++++++++++++++++ yarnlib/scenario_validator_tests.py | 68 ++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 35 deletions(-) create mode 100644 yarnlib/scenario_validator.py create mode 100644 yarnlib/scenario_validator_tests.py diff --git a/yarn b/yarn index af25d21..f982e77 100755 --- a/yarn +++ b/yarn @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2013 Lars Wirzenius +# Copyright 2013-2014 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -119,9 +119,8 @@ class YarnRunner(cliapp.Application): '%String(step_name)') scenarios, implementations = self.parse_scenarios(args) - self.check_there_are_scenarios(scenarios) - self.check_for_duplicate_scenario_names(scenarios) - self.check_for_thens(scenarios) + sv = yarnlib.ScenarioValidator(scenarios) + sv.validate_all() scenarios = self.connect_implementations(scenarios, implementations) shell_prelude = self.load_shell_libraries() @@ -178,36 +177,6 @@ class YarnRunner(cliapp.Application): return block_parser.scenarios, block_parser.implementations - def check_there_are_scenarios(self, scenarios): - if not scenarios: - raise cliapp.AppException( - 'There are no scenarios; must have at least one.') - - def check_for_duplicate_scenario_names(self, scenarios): - counts = collections.Counter() - for s in scenarios: - counts[s.name] += 1 - - duplicates = [name for name in counts if counts[name] > 1] - if duplicates: - duplist = ''.join(' %s\n' % name for name in duplicates) - raise cliapp.AppException( - 'There are scenarios with duplicate names:\n%s' % duplist) - - def check_for_thens(self, scenarios): - no_thens = [] - for scenario in scenarios: - for step in scenario.steps: - if step.what == 'THEN': - break - else: - no_thens.append(scenario) - - if no_thens: - raise cliapp.AppException( - 'Some scenarios have no THENs:\n%s' % - ''.join(' "%s"\n' % s.name for s in no_thens)) - def connect_implementations(self, scenarios, implementations): new_list = [] for scenario in scenarios: diff --git a/yarnlib/__init__.py b/yarnlib/__init__.py index 88bf46f..f224a0c 100644 --- a/yarnlib/__init__.py +++ b/yarnlib/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2013 Lars Wirzenius +# Copyright 2013-2014 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,3 +19,5 @@ from mdparser import MarkdownParser from elements import Scenario, ScenarioStep, Implementation from block_parser import BlockParser, BlockError +from scenario_validator import (NoScenariosError, DuplicateScenariosError, + NoThensError, ScenarioValidator) diff --git a/yarnlib/scenario_validator.py b/yarnlib/scenario_validator.py new file mode 100644 index 0000000..59c86e7 --- /dev/null +++ b/yarnlib/scenario_validator.py @@ -0,0 +1,82 @@ +# Copyright 2014 Lars Wirzenius and Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import cliapp +import collections + + +class NoScenariosError(cliapp.AppException): + + def __init__(self): + cliapp.AppException.__init__( + self, 'There are no scenarios; must have at least one.') + + +class DuplicateScenariosError(cliapp.AppException): + + def __init__(self, duplicates): + duplist = ''.join(' %s\n' % name for name in duplicates) + cliapp.AppException.__init__( + self, 'There are scenarios with duplicate names:\n%s' % duplist) + + +class NoThensError(cliapp.AppException): + + def __init__(self, thenless): + cliapp.AppException.__init__( + self, 'Some scenarios have no THENs:\n%s' % + ''.join(' "%s"\n' % s.name for s in thenless)) + + +class ScenarioValidator(object): + + def __init__(self, scenarios): + self.scenarios = scenarios + + def validate_all(self): # pragma: no cover + '''Convenience method to run all checks.''' + self.check_there_are_scenarios() + self.check_for_duplicate_scenario_names() + self.check_for_thens() + + def check_there_are_scenarios(self): + '''Check whether any scenarios were found.''' + if not self.scenarios: + raise NoScenariosError() + + def check_for_duplicate_scenario_names(self): + '''Check whether multiple scenarios were found sharing a name.''' + counts = collections.Counter() + for s in self.scenarios: + counts[s.name] += 1 + + duplicates = [name for name in counts if counts[name] > 1] + if duplicates: + raise DuplicateScenariosError(duplicates) + + def check_for_thens(self): + no_thens = [] + for scenario in self.scenarios: + for step in scenario.steps: + if step.what == 'THEN': + break + else: + no_thens.append(scenario) + + if no_thens: + raise NoThensError(no_thens) diff --git a/yarnlib/scenario_validator_tests.py b/yarnlib/scenario_validator_tests.py new file mode 100644 index 0000000..1e482a5 --- /dev/null +++ b/yarnlib/scenario_validator_tests.py @@ -0,0 +1,68 @@ +# Copyright 2014 Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import unittest + +import yarnlib + + +class ScenarioValidatorTests(unittest.TestCase): + + def setUp(self): + pass + + def test_check_scenarios_fail(self): + sv = yarnlib.ScenarioValidator([]) + with self.assertRaises(yarnlib.NoScenariosError): + sv.check_there_are_scenarios() + + def test_check_scenarios_pass(self): + sv = yarnlib.ScenarioValidator([ + yarnlib.Scenario('foo') + ]) + sv.check_there_are_scenarios() + + def test_check_duplicate_names_fail(self): + sv = yarnlib.ScenarioValidator([ + yarnlib.Scenario('foo'), + yarnlib.Scenario('foo'), + ]) + with self.assertRaises(yarnlib.DuplicateScenariosError): + sv.check_for_duplicate_scenario_names() + + def test_check_duplicate_names_pass(self): + sv = yarnlib.ScenarioValidator([ + yarnlib.Scenario('foo'), + yarnlib.Scenario('bar'), + ]) + sv.check_for_duplicate_scenario_names() + + def test_check_for_thens_fail(self): + sv = yarnlib.ScenarioValidator([ + yarnlib.Scenario('foo') + ]) + with self.assertRaises(yarnlib.NoThensError): + sv.check_for_thens() + + def test_check_for_thens_pass(self): + scenario = yarnlib.Scenario('foo') + scenario.steps = [ + yarnlib.ScenarioStep('THEN', 'badger stoat') + ] + sv = yarnlib.ScenarioValidator([scenario]) + sv.check_for_thens() -- cgit v1.2.1 From 5e4610f0a687f4b308ac1283d4c2ad0b707e42f5 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 5 Feb 2014 14:31:10 +0000 Subject: Factor out shell library loading into yarnlib --- yarn | 11 ++------ yarnlib/__init__.py | 2 ++ yarnlib/shell_libraries.py | 45 ++++++++++++++++++++++++++++++ yarnlib/shell_libraries_tests.py | 59 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 yarnlib/shell_libraries.py create mode 100644 yarnlib/shell_libraries_tests.py diff --git a/yarn b/yarn index f982e77..cc794f7 100755 --- a/yarn +++ b/yarn @@ -222,14 +222,9 @@ class YarnRunner(cliapp.Application): self.info('No shell libraries defined') return '' - libs = [] - for filename in self.settings['shell-library']: - self.info('Loading shell library %s' % filename) - with open(filename) as f: - text = f.read() - libs.append('# Loaded from %s\n\n%s\n\n' % (filename, text)) - - return ''.join(libs) + return yarnlib.load_shell_libraries( + self.settings['shell-library'], + pre_read_cb=lambda fn: self.info('Loading shell library %s' % fn)) def select_scenarios(self, scenarios): diff --git a/yarnlib/__init__.py b/yarnlib/__init__.py index f224a0c..b5b207d 100644 --- a/yarnlib/__init__.py +++ b/yarnlib/__init__.py @@ -21,3 +21,5 @@ from elements import Scenario, ScenarioStep, Implementation from block_parser import BlockParser, BlockError from scenario_validator import (NoScenariosError, DuplicateScenariosError, NoThensError, ScenarioValidator) + +from shell_libraries import load_shell_libraries diff --git a/yarnlib/shell_libraries.py b/yarnlib/shell_libraries.py new file mode 100644 index 0000000..3607adc --- /dev/null +++ b/yarnlib/shell_libraries.py @@ -0,0 +1,45 @@ +# Copyright 2014 Lars Wirzenius and Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +def load_shell_libraries(paths, pre_read_cb=lambda filename: None, open=open): + '''Helper for loading shell libraries from files. + + `paths`: Iterable of file paths to shell libraries. + `pre_read_cb`: Optional callback for providing progress reporting + and logging. + It is given only one parameter, of the path to the + shell library that is about to be loaded. + `open`: Optional function for opening file paths. This is useful to + override how files are opened for unit tests or virtual + file systems. + + This function is provided so that when shell libraries are loaded + by applications that use yarnlib, they can all load them with the + same format, including the comments about where the snippet came from, + and any future additions. + + ''' + + libs = [] + for filename in paths: + pre_read_cb(filename) + with open(filename) as f: + text = f.read() + libs.append('# Loaded from %s\n\n%s\n\n' % (filename, text)) + + return ''.join(libs) diff --git a/yarnlib/shell_libraries_tests.py b/yarnlib/shell_libraries_tests.py new file mode 100644 index 0000000..69e015a --- /dev/null +++ b/yarnlib/shell_libraries_tests.py @@ -0,0 +1,59 @@ +# Copyright 2014 Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import contextlib +import StringIO +import unittest + +import yarnlib + + +class ShellLibrariesTests(unittest.TestCase): + + libfiles = { + 'die.shlib': ''' + die() { + echo "$@" >&2 + return 1 + } + ''', + 'chdir.shlib':''' + chdir() { + ( + cd "$1" + shift + exec "$@" + ) + } + ''', + } + + def fakeopen(self, path, *args, **kwargs): + return contextlib.closing(StringIO.StringIO(self.libfiles[path])) + + def test_file_names_listed(self): + prelude = yarnlib.load_shell_libraries(self.libfiles.keys(), + open=self.fakeopen) + for filename in self.libfiles.iterkeys(): + self.assertIn(filename, prelude) + + def test_all_contents_included(self): + prelude = yarnlib.load_shell_libraries(self.libfiles.keys(), + open=self.fakeopen) + for contents in self.libfiles.itervalues(): + self.assertIn(contents, prelude) -- cgit v1.2.1 From 711c89a85b52a987ea284d905c41bd5b52e5374a Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 5 Feb 2014 15:37:01 +0000 Subject: Factor scenario step and implementation connector into yarnlib Rather than passing allow-missing-steps in, `missing_step_cb` can be passed to the ScenarioStepConnector, which if it returns true, will exclude that scenario, rather than raising an exception. This was chosen, as it also works as a status update callback. --- yarn | 56 +++------------- yarnlib/__init__.py | 5 ++ yarnlib/scenario_step_connector.py | 108 +++++++++++++++++++++++++++++++ yarnlib/scenario_step_connector_tests.py | 95 +++++++++++++++++++++++++++ 4 files changed, 217 insertions(+), 47 deletions(-) create mode 100644 yarnlib/scenario_step_connector.py create mode 100644 yarnlib/scenario_step_connector_tests.py diff --git a/yarn b/yarn index cc794f7..131ea2d 100755 --- a/yarn +++ b/yarn @@ -178,44 +178,18 @@ class YarnRunner(cliapp.Application): return block_parser.scenarios, block_parser.implementations def connect_implementations(self, scenarios, implementations): - new_list = [] - for scenario in scenarios: - missing_step = False - for step in scenario.steps: - self.connect_implementation( - scenario, step, implementations) - if step.implementation is None: - missing_step = True - if not missing_step: - new_list.append(scenario) - return new_list - - def connect_implementation(self, scenario, step, implementations): - matching = [i for i in implementations - if step.what == i.what and - self.implements_matches_step(i, step)] - - if len(matching) == 0: - if self.settings['allow-missing-steps']: + if self.settings['allow-missing-steps']: + def warn_missing(scenario, step): self.warning( 'Scenario %s has missing step %s %s' % (scenario.name, step.what, step.text)) - return - raise cliapp.AppException( - 'Scenario "%s", step "%s %s" has no matching ' - 'implementation' % - (scenario.name, step.what, step.text)) - if len(matching) > 1: - s = '\n'.join( - 'IMPLEMENTS %s %s' % (i.what, i.regexp) - for i in matching) - raise cliapp.AppException( - 'Scenario "%s", step "%s %s" has more than one ' - 'matching implementations:\n%s' % - (scenario.name, step.what, step.text, s)) + return True + step_connector = yarnlib.ScenarioStepConnector( + implementations, missing_step_cb=warn_missing) + else: + step_connector = yarnlib.ScenarioStepConnector(implementations) - assert step.implementation is None - step.implementation = matching[0] + return step_connector.connect_implementations(scenarios) def load_shell_libraries(self): if not self.settings['shell-library']: @@ -366,7 +340,7 @@ class YarnRunner(cliapp.Application): self.ts['step_name'] = '%s %s' % (step.what, step.text) self.steps_run += 1 - m = self.implements_matches_step(step.implementation, step) + m = yarnlib.implements_matches_step(step.implementation, step) assert m is not None env = self.clean_env() env['DATADIR'] = datadir @@ -435,18 +409,6 @@ class YarnRunner(cliapp.Application): nice = ''.join(nice) return nice - def implements_matches_step(self, implements, step): - '''Return re.Match if implements matches the step. - - Otherwise, return None. - - ''' - - m = re.match(implements.regexp, step.text, re.I) - if m and m.end() != len(step.text): - return None - return m - def indent(self, s): return ''.join(' %s\n' % line for line in s.splitlines()) diff --git a/yarnlib/__init__.py b/yarnlib/__init__.py index b5b207d..e17223a 100644 --- a/yarnlib/__init__.py +++ b/yarnlib/__init__.py @@ -21,5 +21,10 @@ from elements import Scenario, ScenarioStep, Implementation from block_parser import BlockParser, BlockError from scenario_validator import (NoScenariosError, DuplicateScenariosError, NoThensError, ScenarioValidator) +from scenario_step_connector import (implements_matches_step, + ScenarioStepConnector, + StepNotImplementedError, + StepMultipleImplementationsError) + from shell_libraries import load_shell_libraries diff --git a/yarnlib/scenario_step_connector.py b/yarnlib/scenario_step_connector.py new file mode 100644 index 0000000..3e0712f --- /dev/null +++ b/yarnlib/scenario_step_connector.py @@ -0,0 +1,108 @@ +# Copyright 2014 Lars Wirzenius and Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import cliapp +import re + + +def implements_matches_step(implements, step): + '''Return re.Match if implements matches the step. + + Otherwise, return None. + + ''' + + m = re.match(implements.regexp, step.text, re.I) + if m and m.end() != len(step.text): + return None + return m + + +class StepNotImplementedError(cliapp.AppException): + + def __init__(self, scenario, step): + cliapp.AppException.__init__( + self, 'Scenario "%s", step "%s %s" has no matching ' + 'implementation' % + (scenario.name, step.what, step.text)) + + +class StepMultipleImplementationsError(cliapp.AppException): + + def __init__(self, scenario, step, matching): + s = '\n'.join( + 'IMPLEMENTS %s %s' % (i.what, i.regexp) + for i in matching) + cliapp.AppException.__init__( + self, 'Scenario "%s", step "%s %s" has more than one ' + 'matching implementations:\n%s' % + (scenario.name, step.what, step.text, s)) + + +class ScenarioStepConnector(object): + '''Connect scenario steps to their implementations.''' + + def __init__(self, implementations, missing_step_cb=lambda sc, st: None): + self.implementations = implementations + self.missing_step_cb = missing_step_cb + + def connect_implementations(self, scenarios): + '''Connect scenario steps, returning implemented scenarios. + + For each scenario given, connect known implementations to + the steps. + + This returns only scenarios that were fully implemented. + Unless ``missing_step_cb`` is provided and returns true for all + missing steps, an exception is raised instead. + + ''' + implemented_scenarios = [] + for scenario in scenarios: + missing_step = False + for step in scenario.steps: + self.connect_implementation(scenario, step) + if step.implementation is None: + missing_step = True + if not missing_step: + implemented_scenarios.append(scenario) + return implemented_scenarios + + def connect_implementation(self, scenario, step): + '''Connect the step of a scenario to its implementation. + + Always raises an exception if there are multiple matching + implementations. + + Raises an exception if there are no matching implementations + unless ``missing_step_cb`` returns true. + + ''' + matching = [i for i in self.implementations + if step.what == i.what and + implements_matches_step(i, step)] + + if len(matching) == 0: + if self.missing_step_cb(scenario, step): + return + raise StepNotImplementedError(scenario, step) + if len(matching) > 1: + raise StepMultipleImplementationsError(scenario, step, matching) + + assert step.implementation is None + step.implementation = matching[0] diff --git a/yarnlib/scenario_step_connector_tests.py b/yarnlib/scenario_step_connector_tests.py new file mode 100644 index 0000000..e93d904 --- /dev/null +++ b/yarnlib/scenario_step_connector_tests.py @@ -0,0 +1,95 @@ +# Copyright 2014 Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import unittest + +import yarnlib + + +class ImplementsMatchesTests(unittest.TestCase): + + def test_match(self): + i = yarnlib.Implementation('THEN', r'foo', 'echo foo\n') + step = yarnlib.ScenarioStep('THEN', 'foo') + self.assertTrue(yarnlib.implements_matches_step(i, step)) + + def test_no_match(self): + i = yarnlib.Implementation('THEN', r'foo', 'echo foo\n') + step = yarnlib.ScenarioStep('THEN', 'bar') + self.assertIs(yarnlib.implements_matches_step(i, step), None) + + def test_not_match_all(self): + i = yarnlib.Implementation('THEN', r'foo', 'echo foo\n') + step = yarnlib.ScenarioStep('THEN', 'foo bar') + self.assertIs(yarnlib.implements_matches_step(i, step), None) + + +class ScenarioStepConnectorTests(unittest.TestCase): + + def setUp(self): + pass + + def test_all_implemented(self): + impl = yarnlib.Implementation('THEN', r'foo', 'echo foo\n') + scenario = yarnlib.Scenario('foo') + scenario.steps = [ + yarnlib.ScenarioStep('THEN', 'foo') + ] + ssc = yarnlib.ScenarioStepConnector([impl]) + scenarios = ssc.connect_implementations([scenario]) + self.assertIs(scenarios[0].steps[0].implementation, impl) + + def test_multiple_impls(self): + ssc = yarnlib.ScenarioStepConnector([ + yarnlib.Implementation('THEN', r'foo \S+', 'echo foo\n'), + yarnlib.Implementation('THEN', r'\S+ bar', 'echo bar\n'), + ]) + scenario = yarnlib.Scenario('foo bar') + scenario.steps = [ + yarnlib.ScenarioStep('THEN', 'foo bar') + ] + with self.assertRaises(yarnlib.StepMultipleImplementationsError): + scenarios = ssc.connect_implementations([scenario]) + + def test_noimpl_fail(self): + ssc = yarnlib.ScenarioStepConnector([ + yarnlib.Implementation('THEN', r'foo', 'echo foo\n'), + ]) + scenarios = [] + for scenario_name, step_text in [('foo', 'foo'), ('bar', 'bar')]: + scenario = yarnlib.Scenario(scenario_name) + scenario.steps = [ + yarnlib.ScenarioStep('THEN', step_text), + ] + scenarios.append(scenario) + with self.assertRaises(yarnlib.StepNotImplementedError): + scenarios = ssc.connect_implementations(scenarios) + + def test_noimpl_ignore(self): + ssc = yarnlib.ScenarioStepConnector([ + yarnlib.Implementation('THEN', r'foo', 'echo foo\n'), + ], lambda *args: True) + all_scenarios = [] + for scenario_name, step_text in [('foo', 'foo'), ('bar', 'bar')]: + scenario = yarnlib.Scenario(scenario_name) + scenario.steps = [ + yarnlib.ScenarioStep('THEN', step_text), + ] + all_scenarios.append(scenario) + implemented_scenarios = ssc.connect_implementations(all_scenarios) + self.assertTrue(implemented_scenarios) -- cgit v1.2.1 From c43e94039a017e60743e49822d932facef54da93 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 6 Feb 2014 12:10:01 +0000 Subject: Factor scenario running logic out into yarnlib There's now ScenarioRunner, which is given the environment to run scenarios in, such as the shell prelude, environment variables and the path to the source tree of the project being tested. This is then given scenarios and the state directories. --- yarn | 151 +++++++++-------------------- yarnlib/__init__.py | 1 + yarnlib/scenario_runner.py | 202 +++++++++++++++++++++++++++++++++++++++ yarnlib/scenario_runner_tests.py | 129 +++++++++++++++++++++++++ 4 files changed, 377 insertions(+), 106 deletions(-) create mode 100644 yarnlib/scenario_runner.py create mode 100644 yarnlib/scenario_runner_tests.py diff --git a/yarn b/yarn index 131ea2d..6dc09c5 100755 --- a/yarn +++ b/yarn @@ -118,6 +118,13 @@ class YarnRunner(cliapp.Application): '%String(scenario_name): ' '%String(step_name)') + if self.settings['tempdir']: + self.tempdir = os.path.abspath(self.settings['tempdir']) + if not os.path.exists(self.tempdir): + os.mkdir(self.tempdir) + else: + self.tempdir = tempfile.mkdtemp() + scenarios, implementations = self.parse_scenarios(args) sv = yarnlib.ScenarioValidator(scenarios) sv.validate_all() @@ -136,13 +143,21 @@ class YarnRunner(cliapp.Application): self.steps_run = 0 self.timings = [] + scenario_runner = yarnlib.ScenarioRunner(shell_prelude, os.getcwd(), + self.parse_env(), + pre_step_cb=self.pre_step, + post_step_cb=self.post_step) + start_time = time.time() failed_scenarios = [] for scenario in self.select_scenarios(scenarios): - if not self.run_scenario(scenario, shell_prelude): + if not self.run_scenario(scenario_runner, scenario): failed_scenarios.append(scenario) duration = time.time() - start_time + if not self.settings['snapshot']: + shutil.rmtree(self.tempdir) + if not self.settings['quiet']: self.ts.clear() self.ts.finish() @@ -219,7 +234,7 @@ class YarnRunner(cliapp.Application): return scenarios - def run_scenario(self, scenario, shell_prelude): + def run_scenario(self, scenario_runner, scenario): self.start_scenario_timing(scenario.name) started = time.time() @@ -234,62 +249,15 @@ class YarnRunner(cliapp.Application): self.remember_scenario_timing(time.time() - started) return True - if self.settings['tempdir']: - tempdir = os.path.abspath(self.settings['tempdir']) - if not os.path.exists(tempdir): - os.mkdir(tempdir) - else: - tempdir = tempfile.mkdtemp() - - os.mkdir(self.scenario_dir(tempdir, scenario)) - datadir = self.datadir(tempdir, scenario) + os.mkdir(self.scenario_dir(self.tempdir, scenario)) + datadir = self.datadir(self.tempdir, scenario) os.mkdir(datadir) self.info('DATADIR is %s' % datadir) homedir = self.homedir(datadir) os.mkdir(homedir) self.info('HOME for tests is %s' % homedir) - assuming = [s for s in scenario.steps if s.what == 'ASSUMING'] - cleanup = [s for s in scenario.steps if s.what == 'FINALLY'] - normal = [s for s in scenario.steps if s not in assuming + cleanup] - - ok = True - step_number = 0 - - for step in assuming: - exit = self.run_step(datadir, scenario, step, shell_prelude, False) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - self.ts.notify( - 'Skipping "%s" because "%s %s" failed' % - (scenario.name, step.what, step.text)) - self.skipped_for_assuming += 1 - break - else: - for step in normal: - exit = self.run_step( - datadir, scenario, step, shell_prelude, True) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - ok = False - break - - for step in cleanup: - exit = self.run_step( - datadir, scenario, step, shell_prelude, True) - step_number += 1 - self.snapshot_datadir( - tempdir, datadir, scenario, step_number, step) - if exit != 0: - ok = False - break - - if not self.settings['snapshot']: - shutil.rmtree(tempdir) + ok = scenario_runner.run_scenario(scenario, datadir, homedir) self.remember_scenario_timing(time.time() - started) return ok @@ -297,42 +265,16 @@ class YarnRunner(cliapp.Application): def homedir(self, datadir): return os.path.join(datadir, 'HOME') - def clean_env(self): - '''Return a clean environment for running tests.''' - - whitelisted = [ - 'PATH', - ] - - hardcoded = { - 'TERM': 'dumb', - 'SHELL': '/bin/sh', - 'LC_ALL': 'C', - 'USER': 'tomjon', - 'USERNAME': 'tomjon', - 'LOGNAME': 'tomjon', - } - - env = {} - - for key in whitelisted: - if key in os.environ: - env[key] = os.environ[key] - - for key in hardcoded: - env[key] = hardcoded[key] - + def parse_env(self): for option_arg in self.settings['env']: if '=' not in option_arg: raise cliapp.AppException( '--env argument must contain "=" ' 'to separate environment variable name and value') key, value = option_arg.split('=', 1) - env[key] = value + yield key, value - return env - - def run_step(self, datadir, scenario, step, shell_prelude, report_error): + def pre_step(self, step, **ignored): started = time.time() self.info('Running step "%s %s"' % (step.what, step.text)) @@ -340,19 +282,12 @@ class YarnRunner(cliapp.Application): self.ts['step_name'] = '%s %s' % (step.what, step.text) self.steps_run += 1 - m = yarnlib.implements_matches_step(step.implementation, step) - assert m is not None - env = self.clean_env() - env['DATADIR'] = datadir - env['SRCDIR'] = os.getcwd() - env['HOME'] = self.homedir(datadir) - for i, match in enumerate(m.groups('')): - env['MATCH_%d' % (i+1)] = match + return (started,) - shell_script = '%s\n\n%s\n' % ( - shell_prelude, step.implementation.shell) - exit, stdout, stderr = cliapp.runcmd_unchecked( - ['sh', '-xeuc', shell_script], env=env) + def post_step(self, scenario, step, step_number, step_env, + exit, stdout, stderr, pre_step_userdata): + stopped = time.time() + (started,) = pre_step_userdata logging.debug('Exit code: %d' % exit) if stdout: @@ -363,20 +298,24 @@ class YarnRunner(cliapp.Application): logging.debug('Standard error:\n%s' % self.indent(stderr)) else: logging.debug('Standard error: empty') - - if exit != 0 and report_error: - self.error( - 'ERROR: In scenario "%s"\nstep "%s %s" failed,\n' - 'with exit code %d:\n' - 'Standard output from shell command:\n%s' - 'Standard error from shell command:\n%s' % - (scenario.name, step.what, step.text, exit, - self.indent(stdout), self.indent(stderr))) - + if exit != 0: + if step.what == 'ASSUMING': + self.ts.notify( + 'Skipping "%s" because "%s %s" failed' % + (scenario.name, step.what, step.text)) + self.skipped_for_assuming += 1 + else: + self.error( + 'ERROR: In scenario "%s"\nstep "%s %s" failed,\n' + 'with exit code %d:\n' + 'Standard output from shell command:\n%s' + 'Standard error from shell command:\n%s' % + (scenario.name, step.what, step.text, exit, + self.indent(stdout), self.indent(stderr))) self.remember_step_timing( - '%s %s' % (step.what, step.text), time.time() - started) - - return exit + '%s %s' % (step.what, step.text), stopped - started) + self.snapshot_datadir(self.tempdir, step_env['DATADIR'], + scenario, step_number, step) def scenario_dir(self, tempdir, scenario): return os.path.join(tempdir, self.nice(scenario.name)) diff --git a/yarnlib/__init__.py b/yarnlib/__init__.py index e17223a..515082e 100644 --- a/yarnlib/__init__.py +++ b/yarnlib/__init__.py @@ -25,6 +25,7 @@ from scenario_step_connector import (implements_matches_step, ScenarioStepConnector, StepNotImplementedError, StepMultipleImplementationsError) +from scenario_runner import ScenarioRunner from shell_libraries import load_shell_libraries diff --git a/yarnlib/scenario_runner.py b/yarnlib/scenario_runner.py new file mode 100644 index 0000000..e0cdf62 --- /dev/null +++ b/yarnlib/scenario_runner.py @@ -0,0 +1,202 @@ +# Copyright 2014 Lars Wirzenius and Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import cliapp +import os + +import yarnlib + + +def default_pre_step(scenario, step, step_number, step_env): + '''Default callback run before each step in ScenarioRunner. + + Parameters: + + * `scenario`: The yarnlib.Scenario that is about to be run. + * `step`: The yarnlib.ScenarioStep that is about to be run. + * `step_number`: Per-scenario counter for the step that is about to be run, + starting from 1. + * `step_env`: Environment that will be used for this step. + * `return`: Any returned values are passed to `post_step_cb`. + + All parameters are passed as keyword-arguments, so the order of + parameters is not important, and unused arguments can be ignored by + putting **kwargs in the parameter definition. + + ''' + pass + + +def default_post_step(scenario, step, step_number, step_env, + exit, stdout, stderr, pre_step_userdata): + '''Default callback run after each step in ScenarioRunner. + + Parameters: + + * `scenario`: The yarnlib.Scenario that has just been run. + * `step`: The yarnlib.ScenarioStep that has just been run. + * `step_number`: Per-scenario counter for the step that is about to be run, + starting from 1. + * `step_env`: Environment that was used for this step. + * `exit`: Return code of the step that was run. + * `stdout`: Standard output of the step that was just run. + * `stderr`: Standard error of the step that was just run. + * `pre_step_userdata`: Return value from `pre_step_cb`. + + All parameters are passed as keyword-arguments, so the order of + parameters is not important, and unused arguments can be ignored by + putting **kwargs in the parameter definition. + + ''' + pass + + +# Arguably this could just be the run_scenario method with all the +# constructor's parameters passed in, but reeks of poor design with that +# many parameters. +# However, decoupling the ScenarioRunner from the scenarios allows the +# same scenarios to be run against different projects that aim to satisfy +# the same requirements, or multiple yarn suites on the same project, +# if the project aims to satisfy multiple sets of requirements. +# While this use is far less likely than having 1 yarn suite per project, +# it's a useful metric for deciding how to split the arguments between the +# constructor and the run_scenario method. +class ScenarioRunner(object): + '''Sets up an environment to run scenarios on a Project. + + Parameters: + + * `shell_prelude`: Prefix to all shell scripts run, e.g. shell libraries. + * `srcdir`: Path to the root of the source tree. + This is available as the $SRCDIR environment variable + and is the directory the scenarios are run from. + * `extra_env`: dict, or iterable of pairs of variables to add to + the clean environment provided for every command. + * `pre_step_cb`: Callback run before each step in a scenario. + See default_pre_step for an explanation of its signature. + Its return value is passed to `post_step_cb`. + * `post_step_cb`: Callback run after each step in a scenario. + See default_post_step for an explanation of its signature. + * `cmdrunner`: Function used to run step commands in place of + `cliapp.runcmd_unchecked`, intended to be replaceable + by unit tests. + + ''' + + + def __init__(self, shell_prelude, srcdir, extra_env=(), + pre_step_cb=default_pre_step, post_step_cb=default_post_step, + cmdrunner=cliapp.runcmd_unchecked): + self.shell_prelude = shell_prelude + self.srcdir = srcdir + self.env = self.clean_env(extra_env, SRCDIR=srcdir) + self.pre_step_cb = pre_step_cb + self.post_step_cb = post_step_cb + self.cmdrunner = cmdrunner + + def run_scenario(self, scenario, datadir, homedir): + assuming = [s for s in scenario.steps if s.what == 'ASSUMING'] + cleanup = [s for s in scenario.steps if s.what == 'FINALLY'] + normal = [s for s in scenario.steps if s not in assuming + cleanup] + + ok = True + step_number = 1 + + scenario_env = dict(self.env) + scenario_env['HOME'] = homedir + scenario_env['DATADIR'] = datadir + + for step in assuming: + exit = self.run_step(scenario, step, scenario_env, step_number) + step_number += 1 + if exit != 0: + break + else: + for step in normal: + exit = self.run_step(scenario, step, scenario_env, step_number) + step_number += 1 + if exit != 0: + ok = False + break + + for step in cleanup: + exit = self.run_step(scenario, step, scenario_env, step_number) + step_number += 1 + if exit != 0: + ok = False + break + + return ok + + def run_step(self, scenario, step, scenario_env, step_number): + m = yarnlib.implements_matches_step(step.implementation, step) + assert m is not None + step_env = dict(scenario_env) + for i, match in enumerate(m.groups('')): + step_env['MATCH_%d' % (i+1)] = match + + # All parameters passed as keyword-arguments, so that the callback + # may declare parameters in any order, and ignore any parameters + # by specifying **kwargs + pre_step_userdata = self.pre_step_cb(scenario=scenario, step=step, + step_number=step_number, + step_env=step_env) + + shell_script = '%s\n\n%s\n' % ( + self.shell_prelude, step.implementation.shell) + exit, stdout, stderr = self.cmdrunner( + ['sh', '-xeuc', shell_script], env=step_env, cwd=self.srcdir) + + # All parameters passed as keyword-arguments, so that the callback + # may declare parameters in any order, and ignore any parameters + # by specifying **kwargs + self.post_step_cb(scenario=scenario, step=step, + step_number=step_number, step_env=step_env, + exit=exit, stdout=stdout, stderr=stderr, + pre_step_userdata=pre_step_userdata) + + return exit + + @staticmethod + def clean_env(extra_env, **kwarg_env): + '''Return a clean environment for running tests.''' + + whitelisted = [ + 'PATH', + ] + + hardcoded = { + 'TERM': 'dumb', + 'SHELL': '/bin/sh', + 'LC_ALL': 'C', + 'USER': 'tomjon', + 'USERNAME': 'tomjon', + 'LOGNAME': 'tomjon', + } + + env = {} + + for key in whitelisted: + if key in os.environ: + env[key] = os.environ[key] + + env.update(hardcoded) + env.update(extra_env) + env.update(kwarg_env) + + return env diff --git a/yarnlib/scenario_runner_tests.py b/yarnlib/scenario_runner_tests.py new file mode 100644 index 0000000..cda6397 --- /dev/null +++ b/yarnlib/scenario_runner_tests.py @@ -0,0 +1,129 @@ +# Copyright 2014 Codethink Limited +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import itertools +import unittest + +import yarnlib + + +class ScenarioRunnerEnvironmentTests(unittest.TestCase): + + def setUp(self): + self.cmdlog = [] + self.scenario = yarnlib.Scenario('foo') + step = yarnlib.ScenarioStep('THEN', 'foo bar') + step.implementation = yarnlib.Implementation('THEN', r'foo (\S+)', + 'echo foo $MATCH_1') + self.scenario.steps = [step] + + def fake_cmdrunner(self, cmd, env=None, cwd=None): + self.cmdlog.append((cmd, env, cwd)) + return 0, '', '' + + def test_command_env(self): + sr = yarnlib.ScenarioRunner('', '/tmp', {'FOO': 'bar'}, + cmdrunner=self.fake_cmdrunner) + sr.run_scenario(self.scenario, '/tmp/datadir', '/tmp/home') + cmd, cmd_env, cmd_cwd = self.cmdlog[0] + self.assertIn('FOO', cmd_env) + self.assertEqual(cmd_env['FOO'], 'bar') + self.assertEqual(cmd_env['SRCDIR'], '/tmp') + self.assertEqual(cmd_env['DATADIR'], '/tmp/datadir') + self.assertEqual(cmd_env['HOME'], '/tmp/home') + self.assertEqual(cmd_env['MATCH_1'], 'bar') + self.assertEqual('/tmp', cmd_cwd) + + def test_command_args(self): + prelude = 'TESTVAR=foo\n' + impl_shell = self.scenario.steps[0].implementation.shell + sr = yarnlib.ScenarioRunner(prelude, '/tmp', + cmdrunner=self.fake_cmdrunner) + sr.run_scenario(self.scenario, '/tmp/datadir', '/tmp/home') + shell, opts, command = self.cmdlog[0][0] + self.assertTrue(shell.endswith('sh')) + self.assertTrue(command.startswith(prelude)) + self.assertIn(impl_shell, command) + + +class ScenarioRunnerFlowTests(unittest.TestCase): + + def setUp(self): + self.cmdlog = [] + self.scenario_runner = ( + yarnlib.ScenarioRunner('', 'srcdir', + cmdrunner=self.fake_cmdrunner)) + for verb, cmd in itertools.product(('ASSUMING', 'THEN', 'FINALLY'), + ('true', 'false')): + step = yarnlib.ScenarioStep(verb, cmd) + step.implementation = yarnlib.Implementation(verb, cmd, cmd) + setattr(self, '%s_%s' % (verb.lower(), cmd), step) + + def fake_cmdrunner(self, cmd, *args, **kwargs): + self.cmdlog.append(cmd) + if 'false' in cmd[-1]: + return 1, '', '' + return 0, '', '' + + def run_scenario(self, *steps): + scenario = yarnlib.Scenario('foo') + scenario.steps = steps + return self.scenario_runner.run_scenario(scenario, 'data', 'home') + + def test_assuming_skips_remaining(self): + ok = self.run_scenario( + self.assuming_false, + self.then_true, + self.finally_true, + ) + self.assertTrue(ok) + self.assertEqual(len(self.cmdlog), 1) + + def test_cleanup_run_on_success(self): + ok = self.run_scenario( + self.assuming_true, + self.then_true, + self.finally_true, + ) + self.assertTrue(ok) + self.assertEqual(len(self.cmdlog), 3) + + def test_cleanup_run_on_failure(self): + ok = self.run_scenario( + self.then_false, + self.finally_true, + ) + self.assertFalse(ok) + self.assertEqual(len(self.cmdlog), 2) + + def test_skip_steps_after_failure(self): + ok = self.run_scenario( + self.then_false, + self.then_true, + ) + self.assertFalse(ok) + self.assertEqual(len(self.cmdlog), 1) + + def test_cleanup_failure(self): + ok = self.run_scenario( + self.then_true, + self.finally_false, + self.finally_true, + ) + self.assertFalse(ok) + self.assertEqual(len(self.cmdlog), 2) -- cgit v1.2.1