summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--README.rst4
-rw-r--r--doc/source/examples.rst12
-rw-r--r--doc/source/img/retry_states.svg6
-rw-r--r--doc/source/img/task_states.svg4
-rw-r--r--doc/source/utils.rst5
-rw-r--r--requirements.txt6
-rw-r--r--taskflow/conductors/base.py6
-rw-r--r--taskflow/engines/action_engine/actions/base.py4
-rw-r--r--taskflow/engines/action_engine/actions/retry.py4
-rw-r--r--taskflow/engines/action_engine/actions/task.py4
-rw-r--r--taskflow/engines/action_engine/analyzer.py49
-rw-r--r--taskflow/engines/action_engine/compiler.py5
-rw-r--r--taskflow/engines/action_engine/engine.py11
-rw-r--r--taskflow/engines/action_engine/runner.py94
-rw-r--r--taskflow/engines/action_engine/runtime.py111
-rw-r--r--taskflow/engines/action_engine/scheduler.py47
-rw-r--r--taskflow/engines/worker_based/protocol.py4
-rw-r--r--taskflow/examples/job_board_no_test.py171
-rw-r--r--taskflow/examples/simple_linear_listening.py4
-rw-r--r--taskflow/jobs/backends/impl_zookeeper.py6
-rw-r--r--taskflow/persistence/backends/impl_dir.py8
-rw-r--r--taskflow/persistence/backends/impl_memory.py23
-rw-r--r--taskflow/retry.py31
-rw-r--r--taskflow/states.py41
-rw-r--r--taskflow/tests/test_examples.py2
-rw-r--r--taskflow/tests/unit/action_engine/test_runner.py26
-rw-r--r--taskflow/tests/unit/persistence/test_sql_persistence.py111
-rw-r--r--taskflow/tests/unit/test_check_transition.py17
-rw-r--r--taskflow/tests/unit/test_retries.py251
-rw-r--r--taskflow/tests/unit/test_utils_lock_utils.py281
-rw-r--r--taskflow/utils/lock_utils.py207
-rw-r--r--test-requirements.txt2
-rwxr-xr-xtools/state_graph.py17
33 files changed, 599 insertions, 975 deletions
diff --git a/README.rst b/README.rst
index 70ff715..9023dd6 100644
--- a/README.rst
+++ b/README.rst
@@ -1,11 +1,11 @@
TaskFlow
========
-.. image:: https://pypip.in/version/taskflow/badge.svg
+.. image:: https://img.shields.io/pypi/v/taskflow.svg
:target: https://pypi.python.org/pypi/taskflow/
:alt: Latest Version
-.. image:: https://pypip.in/download/taskflow/badge.svg?period=month
+.. image:: https://img.shields.io/pypi/dm/taskflow.svg
:target: https://pypi.python.org/pypi/taskflow/
:alt: Downloads
diff --git a/doc/source/examples.rst b/doc/source/examples.rst
index 64229f1..187171f 100644
--- a/doc/source/examples.rst
+++ b/doc/source/examples.rst
@@ -34,6 +34,18 @@ Using listeners
:linenos:
:lines: 16-
+Using listeners (to watch a phone call)
+=======================================
+
+.. note::
+
+ Full source located at :example:`simple_linear_listening`.
+
+.. literalinclude:: ../../taskflow/examples/simple_linear_listening.py
+ :language: python
+ :linenos:
+ :lines: 16-
+
Dumping a in-memory backend
===========================
diff --git a/doc/source/img/retry_states.svg b/doc/source/img/retry_states.svg
index 8b0c635..1a25bda 100644
--- a/doc/source/img/retry_states.svg
+++ b/doc/source/img/retry_states.svg
@@ -1,8 +1,8 @@
<?xml version="1.0"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
-<!-- Generated by graphviz version 2.34.0 (20140110.0949)
+<!-- Generated by graphviz version 2.36.0 (20140111.2315)
-->
<!-- Title: Retries states Pages: 1 -->
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg" width="644pt" height="167pt" viewBox="0.00 0.00 643.60 167.00" preserveAspectRatio="xMidYMid meet" zoomAndPan="magnify" version="1.1" contentScriptType="application/ecmascript" contentStyleType="text/css"><defs><linearGradient id="white" x1="0%" y1="0%" x2="0%" y2="0%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/></linearGradient><linearGradient id="black" x1="0%" y1="0%" x2="0%" y2="0%"><stop offset="0%" style="stop-color:rgb(0,0,0);stop-opacity:1"/></linearGradient><linearGradient id="aquamarine" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(127,255,212);stop-opacity:1"/></linearGradient><linearGradient id="azure" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(240,255,255);stop-opacity:1"/></linearGradient><linearGradient id="blue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,0,255);stop-opacity:1"/></linearGradient><linearGradient id="blueviolet" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(138,43,226);stop-opacity:1"/></linearGradient><linearGradient id="brown" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(165,42,42);stop-opacity:1"/></linearGradient><linearGradient id="cadetblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(95,158,160);stop-opacity:1"/></linearGradient><linearGradient id="chocolate" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(210,105,30);stop-opacity:1"/></linearGradient><linearGradient id="cornflowerblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(100,149,237);stop-opacity:1"/></linearGradient><linearGradient id="crimson" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(220,20,60);stop-opacity:1"/></linearGradient><linearGradient id="cyan" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,255,255);stop-opacity:1"/></linearGradient><linearGradient id="darkgreen" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,100,0);stop-opacity:1"/></linearGradient><linearGradient id="darkorange" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,140,0);stop-opacity:1"/></linearGradient><linearGradient id="gold" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,215,0);stop-opacity:1"/></linearGradient><linearGradient id="gray" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(192,192,192);stop-opacity:1"/></linearGradient><linearGradient id="greenyellow" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(173,255,47);stop-opacity:1"/></linearGradient><linearGradient id="green" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,255,0);stop-opacity:1"/></linearGradient><linearGradient id="grey" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(192,192,192);stop-opacity:1"/></linearGradient><linearGradient id="hotpink" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,105,180);stop-opacity:1"/></linearGradient><linearGradient id="indianred" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(205,92,92);stop-opacity:1"/></linearGradient><linearGradient id="indigo" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(75,0,130);stop-opacity:1"/></linearGradient><linearGradient id="lavender" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(230,230,250);stop-opacity:1"/></linearGradient><linearGradient id="lightblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(173,216,230);stop-opacity:1"/></linearGradient><linearGradient id="lightgray" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(211,211,211);stop-opacity:1"/></linearGradient><linearGradient id="lightgrey" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(211,211,211);stop-opacity:1"/></linearGradient><linearGradient id="magenta" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,0,255);stop-opacity:1"/></linearGradient><linearGradient id="maroon" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(176,48,96);stop-opacity:1"/></linearGradient><linearGradient id="mediumblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,0,205);stop-opacity:1"/></linearGradient><linearGradient id="mediumpurple" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(147,112,219);stop-opacity:1"/></linearGradient><linearGradient id="orange" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,165,0);stop-opacity:1"/></linearGradient><linearGradient id="pink" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,192,203);stop-opacity:1"/></linearGradient><linearGradient id="purple" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(160,32,240);stop-opacity:1"/></linearGradient><linearGradient id="red" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,0,0);stop-opacity:1"/></linearGradient><linearGradient id="steelblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(70,130,180);stop-opacity:1"/></linearGradient><linearGradient id="violet" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(238,130,238);stop-opacity:1"/></linearGradient><linearGradient id="yellow" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,255,0);stop-opacity:1"/></linearGradient><linearGradient id="none" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,255,255);stop-opacity:1"/></linearGradient></defs>
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 163)"><polygon fill="white" stroke="white" points="-4,4 -4,-163 639.6,-163 639.6,4 -4,4"/><title>Retries states</title><g id="node1" class="node"><ellipse fill="none" stroke="black" cx="103.6" cy="-133" rx="39.8775" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="103.6" cy="-133" rx="39.8775" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="103.6" y="-130.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:Verdana">PENDING</text></g><g id="node2" class="node"><ellipse fill="none" stroke="black" cx="221.6" cy="-91" rx="41.4846" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="221.6" cy="-91" rx="41.4846" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="221.6" y="-88.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:Verdana">RUNNING</text></g><g id="edge1" class="edge"><polygon fill="black" stroke="black" points="180.902,-109.096 189.113,-102.4 178.519,-102.514 180.902,-109.096" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M135.078,-121.965C148.604,-117.068 164.729,-111.229 179.314,-105.949" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="180.902,-109.096 189.113,-102.4 178.519,-102.514 180.902,-109.096" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M135.078,-121.965C148.604,-117.068 164.729,-111.229 179.314,-105.949"/></g><g id="node3" class="node"><ellipse fill="none" stroke="black" cx="338.6" cy="-72" rx="39.1741" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="338.6" cy="-72" rx="39.1741" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="338.6" y="-69.2" font-family="Times,serif" font-size="11.00" fill="green" style="font-size:10px; font-family:Verdana">SUCCESS</text></g><g id="edge5" class="edge"><polygon fill="black" stroke="black" points="292.321,-83.0285 301.616,-77.9451 291.179,-76.1221 292.321,-83.0285" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M260.574,-84.726C270.486,-83.0884 281.276,-81.3057 291.557,-79.6072" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="292.321,-83.0285 301.616,-77.9451 291.179,-76.1221 292.321,-83.0285" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M260.574,-84.726C270.486,-83.0884 281.276,-81.3057 291.557,-79.6072"/></g><g id="node6" class="node"><ellipse fill="none" stroke="black" cx="590.6" cy="-72" rx="38.4712" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="590.6" cy="-72" rx="38.4712" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="590.6" y="-69.2" font-family="Times,serif" font-size="11.00" fill="red" style="font-size:10px; font-family:Verdana">FAILURE</text></g><g id="edge6" class="edge"><polygon fill="black" stroke="black" points="553.498,-90.6751 561.664,-83.925 551.071,-84.1093 553.498,-90.6751" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M261.444,-96.7194C316.907,-103.839 421.801,-113.417 509.6,-99 523.778,-96.672 538.826,-92.1687 552.001,-87.497" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="553.498,-90.6751 561.664,-83.925 551.071,-84.1093 553.498,-90.6751" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M261.444,-96.7194C316.907,-103.839 421.801,-113.417 509.6,-99 523.778,-96.672 538.826,-92.1687 552.001,-87.497"/></g><g id="node4" class="node"><ellipse fill="none" stroke="black" cx="461.6" cy="-18" rx="44.498" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="461.6" cy="-18" rx="44.498" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="461.6" y="-15.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:Verdana">RETRYING</text></g><g id="edge2" class="edge"><polygon fill="black" stroke="black" points="423.149,-38.5463 430.855,-31.2747 420.296,-32.154 423.149,-38.5463" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M367.36,-59.6113C383.373,-52.4651 403.764,-43.3648 421.391,-35.498" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="423.149,-38.5463 430.855,-31.2747 420.296,-32.154 423.149,-38.5463" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M367.36,-59.6113C383.373,-52.4651 403.764,-43.3648 421.391,-35.498"/></g><g id="node5" class="node"><ellipse fill="none" stroke="black" cx="461.6" cy="-72" rx="48.2143" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="461.6" cy="-72" rx="48.2143" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="461.6" y="-69.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:Verdana">REVERTING</text></g><g id="edge3" class="edge"><polygon fill="black" stroke="black" points="403.212,-75.5001 413.212,-72 403.212,-68.5001 403.212,-75.5001" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M377.885,-72C385.947,-72 394.62,-72 403.209,-72" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="403.212,-75.5001 413.212,-72 403.212,-68.5001 403.212,-75.5001" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M377.885,-72C385.947,-72 394.62,-72 403.209,-72"/></g><g id="edge4" class="edge"><polygon fill="black" stroke="black" points="249.789,-67.1719 243.419,-75.6381 253.678,-72.9921 249.789,-67.1719" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M417.454,-20.8612C384.379,-24.0868 338.038,-30.8674 299.6,-45 282.935,-51.1272 265.84,-60.922 252.004,-69.9016" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="249.789,-67.1719 243.419,-75.6381 253.678,-72.9921 249.789,-67.1719" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M417.454,-20.8612C384.379,-24.0868 338.038,-30.8674 299.6,-45 282.935,-51.1272 265.84,-60.922 252.004,-69.9016"/></g><g id="edge9" class="edge"><polygon fill="black" stroke="black" points="543.885,-68.9952 554.007,-65.8664 544.143,-61.9999 543.885,-68.9952" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M506.662,-65.5393C518.631,-65.1724 531.619,-65.1545 543.687,-65.4856" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="543.885,-68.9952 554.007,-65.8664 544.143,-61.9999 543.885,-68.9952" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M506.662,-65.5393C518.631,-65.1724 531.619,-65.1545 543.687,-65.4856"/></g><g id="node7" class="node"><ellipse fill="none" stroke="black" cx="590.6" cy="-141" rx="45.2009" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="590.6" cy="-141" rx="45.2009" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="590.6" y="-138.2" font-family="Times,serif" font-size="11.00" fill="darkorange" style="font-size:10px; font-family:Verdana">REVERTED</text></g><g id="edge10" class="edge"><polygon fill="black" stroke="black" points="552.378,-124.76 562.836,-126.459 555.72,-118.61 552.378,-124.76" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M489.845,-86.8022C508.553,-96.9667 533.576,-110.562 553.879,-121.593" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="552.378,-124.76 562.836,-126.459 555.72,-118.61 552.378,-124.76" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M489.845,-86.8022C508.553,-96.9667 533.576,-110.562 553.879,-121.593"/></g><g id="edge8" class="edge"><polygon fill="black" stroke="black" points="516.737,-75.1813 506.662,-78.4607 516.583,-82.1796 516.737,-75.1813" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M554.007,-78.1336C542.529,-78.6746 529.533,-78.8595 517.018,-78.6883" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="516.737,-75.1813 506.662,-78.4607 516.583,-82.1796 516.737,-75.1813" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M554.007,-78.1336C542.529,-78.6746 529.533,-78.8595 517.018,-78.6883"/></g><g id="edge7" class="edge"><polygon fill="black" stroke="black" points="153.526,-130.307 143.47,-133.641 153.411,-137.306 153.526,-130.307" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M545.34,-140.27C454.633,-138.774 248.233,-135.369 153.626,-133.809" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="153.526,-130.307 143.47,-133.641 153.411,-137.306 153.526,-130.307" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M545.34,-140.27C454.633,-138.774 248.233,-135.369 153.626,-133.809"/></g><g id="node8" class="node"><ellipse fill="black" stroke="black" cx="23.6" cy="-133" rx="3.6" ry="3.6" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="black" stroke="black" cx="23.6" cy="-133" rx="3.6" ry="3.6" style="fill:url(#black);stroke:black;"/><text text-anchor="middle" x="10" y="-120.6" font-family="Times,serif" font-size="11.00" fill="green" style="font-size:10px; font-family:Verdana">start</text></g><g id="edge11" class="edge"><polygon fill="black" stroke="black" points="53.5965,-136.5 63.5964,-133 53.5964,-129.5 53.5965,-136.5" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" stroke-dasharray="1,5" d="M27.5624,-133C32.3405,-133 42.3529,-133 53.5516,-133" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="53.5965,-136.5 63.5964,-133 53.5964,-129.5 53.5965,-136.5" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" stroke-dasharray="1,5" d="M27.5624,-133C32.3405,-133 42.3529,-133 53.5516,-133"/></g></g>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg" width="644pt" height="159pt" viewBox="0.00 0.00 643.60 159.00" preserveAspectRatio="xMidYMid meet" zoomAndPan="magnify" version="1.1" contentScriptType="application/ecmascript" contentStyleType="text/css"><defs><linearGradient id="white" x1="0%" y1="0%" x2="0%" y2="0%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/></linearGradient><linearGradient id="black" x1="0%" y1="0%" x2="0%" y2="0%"><stop offset="0%" style="stop-color:rgb(0,0,0);stop-opacity:1"/></linearGradient><linearGradient id="aquamarine" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(127,255,212);stop-opacity:1"/></linearGradient><linearGradient id="azure" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(240,255,255);stop-opacity:1"/></linearGradient><linearGradient id="blue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,0,255);stop-opacity:1"/></linearGradient><linearGradient id="blueviolet" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(138,43,226);stop-opacity:1"/></linearGradient><linearGradient id="brown" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(165,42,42);stop-opacity:1"/></linearGradient><linearGradient id="cadetblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(95,158,160);stop-opacity:1"/></linearGradient><linearGradient id="chocolate" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(210,105,30);stop-opacity:1"/></linearGradient><linearGradient id="cornflowerblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(100,149,237);stop-opacity:1"/></linearGradient><linearGradient id="crimson" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(220,20,60);stop-opacity:1"/></linearGradient><linearGradient id="cyan" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,255,255);stop-opacity:1"/></linearGradient><linearGradient id="darkgreen" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,100,0);stop-opacity:1"/></linearGradient><linearGradient id="darkorange" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,140,0);stop-opacity:1"/></linearGradient><linearGradient id="gold" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,215,0);stop-opacity:1"/></linearGradient><linearGradient id="gray" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(192,192,192);stop-opacity:1"/></linearGradient><linearGradient id="greenyellow" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(173,255,47);stop-opacity:1"/></linearGradient><linearGradient id="green" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,255,0);stop-opacity:1"/></linearGradient><linearGradient id="grey" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(192,192,192);stop-opacity:1"/></linearGradient><linearGradient id="hotpink" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,105,180);stop-opacity:1"/></linearGradient><linearGradient id="indianred" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(205,92,92);stop-opacity:1"/></linearGradient><linearGradient id="indigo" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(75,0,130);stop-opacity:1"/></linearGradient><linearGradient id="lavender" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(230,230,250);stop-opacity:1"/></linearGradient><linearGradient id="lightblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(173,216,230);stop-opacity:1"/></linearGradient><linearGradient id="lightgray" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(211,211,211);stop-opacity:1"/></linearGradient><linearGradient id="lightgrey" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(211,211,211);stop-opacity:1"/></linearGradient><linearGradient id="magenta" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,0,255);stop-opacity:1"/></linearGradient><linearGradient id="maroon" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(176,48,96);stop-opacity:1"/></linearGradient><linearGradient id="mediumblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,0,205);stop-opacity:1"/></linearGradient><linearGradient id="mediumpurple" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(147,112,219);stop-opacity:1"/></linearGradient><linearGradient id="orange" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,165,0);stop-opacity:1"/></linearGradient><linearGradient id="pink" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,192,203);stop-opacity:1"/></linearGradient><linearGradient id="purple" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(160,32,240);stop-opacity:1"/></linearGradient><linearGradient id="red" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,0,0);stop-opacity:1"/></linearGradient><linearGradient id="steelblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(70,130,180);stop-opacity:1"/></linearGradient><linearGradient id="violet" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(238,130,238);stop-opacity:1"/></linearGradient><linearGradient id="yellow" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,255,0);stop-opacity:1"/></linearGradient><linearGradient id="none" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,255,255);stop-opacity:1"/></linearGradient></defs>
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 155)"><polygon fill="white" stroke="none" points="-4,4 -4,-155 639.6,-155 639.6,4 -4,4"/><title>Retries states</title><g id="node1" class="node"><ellipse fill="none" stroke="black" cx="103.6" cy="-18" rx="39.8775" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="103.6" cy="-18" rx="39.8775" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="103.6" y="-15.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:sans-serif;">PENDING</text></g><g id="node2" class="node"><ellipse fill="none" stroke="black" cx="221.6" cy="-56" rx="41.4846" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="221.6" cy="-56" rx="41.4846" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="221.6" y="-53.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:sans-serif;">RUNNING</text></g><g id="edge1" class="edge"><polygon fill="black" stroke="black" points="177.135,-45.4443 187.727,-45.2314 179.314,-38.7922 177.135,-45.4443" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M136.298,-28.3837C149.145,-32.5922 164.176,-37.5163 177.96,-42.0317" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="177.135,-45.4443 187.727,-45.2314 179.314,-38.7922 177.135,-45.4443" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M136.298,-28.3837C149.145,-32.5922 164.176,-37.5163 177.96,-42.0317"/></g><g id="node3" class="node"><ellipse fill="none" stroke="black" cx="338.6" cy="-117" rx="39.1741" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="338.6" cy="-117" rx="39.1741" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="338.6" y="-114.2" font-family="Times,serif" font-size="11.00" fill="green" style="font-size:10px; font-family:sans-serif;">SUCCESS</text></g><g id="edge10" class="edge"><polygon fill="black" stroke="black" points="301.924,-102.038 312.398,-103.632 305.204,-95.8542 301.924,-102.038" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M248.695,-69.8418C264.799,-78.3837 285.643,-89.44 303.1,-98.6999" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="301.924,-102.038 312.398,-103.632 305.204,-95.8542 301.924,-102.038" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M248.695,-69.8418C264.799,-78.3837 285.643,-89.44 303.1,-98.6999"/></g><g id="node6" class="node"><ellipse fill="none" stroke="black" cx="590.6" cy="-133" rx="38.4712" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="590.6" cy="-133" rx="38.4712" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="590.6" y="-130.2" font-family="Times,serif" font-size="11.00" fill="red" style="font-size:10px; font-family:sans-serif;">FAILURE</text></g><g id="edge9" class="edge"><polygon fill="black" stroke="black" points="570.2,-109.624 578.84,-115.756 575.91,-105.574 570.2,-109.624" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M257.86,-46.8398C313.747,-34.1815 424.999,-16.6491 509.6,-52 536.327,-63.1681 558.671,-88.0955 572.892,-107.368" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="570.2,-109.624 578.84,-115.756 575.91,-105.574 570.2,-109.624" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M257.86,-46.8398C313.747,-34.1815 424.999,-16.6491 509.6,-52 536.327,-63.1681 558.671,-88.0955 572.892,-107.368"/></g><g id="node4" class="node"><ellipse fill="none" stroke="black" cx="461.6" cy="-79" rx="44.498" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="461.6" cy="-79" rx="44.498" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="461.6" y="-76.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:sans-serif;">RETRYING</text></g><g id="edge2" class="edge"><polygon fill="black" stroke="black" points="417.556,-96.1866 426.048,-89.8511 415.458,-89.5082 417.556,-96.1866" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M371.391,-107.016C385.177,-102.687 401.55,-97.5445 416.484,-92.8546" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="417.556,-96.1866 426.048,-89.8511 415.458,-89.5082 417.556,-96.1866" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M371.391,-107.016C385.177,-102.687 401.55,-97.5445 416.484,-92.8546"/></g><g id="node5" class="node"><ellipse fill="none" stroke="black" cx="461.6" cy="-133" rx="48.2143" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="461.6" cy="-133" rx="48.2143" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="461.6" y="-130.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:sans-serif;">REVERTING</text></g><g id="edge3" class="edge"><polygon fill="black" stroke="black" points="405.392,-129.23 415.764,-127.071 406.309,-122.291 405.392,-129.23" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M376.561,-121.887C385.743,-123.102 395.782,-124.429 405.623,-125.73" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="405.392,-129.23 415.764,-127.071 406.309,-122.291 405.392,-129.23" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M376.561,-121.887C385.743,-123.102 395.782,-124.429 405.623,-125.73"/></g><g id="edge4" class="edge"><polygon fill="black" stroke="black" points="272.458,-57.302 262.168,-59.8238 271.785,-64.2696 272.458,-57.302" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M418.124,-74.8952C377.203,-70.9406 315.316,-64.96 272.165,-60.7899" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="272.458,-57.302 262.168,-59.8238 271.785,-64.2696 272.458,-57.302" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M418.124,-74.8952C377.203,-70.9406 315.316,-64.96 272.165,-60.7899"/></g><g id="edge7" class="edge"><polygon fill="black" stroke="black" points="543.885,-129.995 554.007,-126.866 544.143,-123 543.885,-129.995" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M506.662,-126.539C518.631,-126.172 531.619,-126.154 543.687,-126.486" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="543.885,-129.995 554.007,-126.866 544.143,-123 543.885,-129.995" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M506.662,-126.539C518.631,-126.172 531.619,-126.154 543.687,-126.486"/></g><g id="node7" class="node"><ellipse fill="none" stroke="black" cx="590.6" cy="-37" rx="45.2009" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="590.6" cy="-37" rx="45.2009" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="590.6" y="-34.2" font-family="Times,serif" font-size="11.00" fill="darkorange" style="font-size:10px; font-family:sans-serif;">REVERTED</text></g><g id="edge6" class="edge"><polygon fill="black" stroke="black" points="567.317,-62.9968 572.288,-53.6408 562.568,-57.8546 567.317,-62.9968" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M489.99,-118.435C496.565,-114.622 503.458,-110.353 509.6,-106 529.096,-92.182 549.47,-74.5566 564.777,-60.5785" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="567.317,-62.9968 572.288,-53.6408 562.568,-57.8546 567.317,-62.9968" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M489.99,-118.435C496.565,-114.622 503.458,-110.353 509.6,-106 529.096,-92.182 549.47,-74.5566 564.777,-60.5785"/></g><g id="edge5" class="edge"><polygon fill="black" stroke="black" points="516.737,-136.181 506.662,-139.461 516.583,-143.18 516.737,-136.181" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M554.007,-139.134C542.529,-139.675 529.533,-139.859 517.018,-139.688" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="516.737,-136.181 506.662,-139.461 516.583,-143.18 516.737,-136.181" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M554.007,-139.134C542.529,-139.675 529.533,-139.859 517.018,-139.688"/></g><g id="edge8" class="edge"><polygon fill="black" stroke="black" points="153.465,-14.3312 143.466,-17.8322 153.466,-21.3312 153.465,-14.3312" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M547.314,-31.5447C535.158,-30.1661 521.871,-28.8401 509.6,-28 381.429,-19.2249 229.976,-17.8811 153.717,-17.8313" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="153.465,-14.3312 143.466,-17.8322 153.466,-21.3312 153.465,-14.3312" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M547.314,-31.5447C535.158,-30.1661 521.871,-28.8401 509.6,-28 381.429,-19.2249 229.976,-17.8811 153.717,-17.8313"/></g><g id="node8" class="node"><ellipse fill="black" stroke="black" cx="23.6" cy="-18" rx="3.6" ry="3.6" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="black" stroke="black" cx="23.6" cy="-18" rx="3.6" ry="3.6" style="fill: url(#black);stroke: black;"/><text text-anchor="middle" x="10" y="-5.6" font-family="Times,serif" font-size="11.00" fill="green" style="font-size:10px; font-family:sans-serif;">start</text></g><g id="edge11" class="edge"><polygon fill="black" stroke="black" points="53.5965,-21.5001 63.5964,-18 53.5964,-14.5001 53.5965,-21.5001" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" stroke-dasharray="1,5" d="M27.5624,-18C32.3405,-18 42.3529,-18 53.5516,-18" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="53.5965,-21.5001 63.5964,-18 53.5964,-14.5001 53.5965,-21.5001" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" stroke-dasharray="1,5" d="M27.5624,-18C32.3405,-18 42.3529,-18 53.5516,-18"/></g></g>
</svg>
diff --git a/doc/source/img/task_states.svg b/doc/source/img/task_states.svg
index 14a1f09..dbb48c6 100644
--- a/doc/source/img/task_states.svg
+++ b/doc/source/img/task_states.svg
@@ -1,8 +1,8 @@
<?xml version="1.0"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
-<!-- Generated by graphviz version 2.34.0 (20140110.0949)
+<!-- Generated by graphviz version 2.36.0 (20140111.2315)
-->
<!-- Title: Tasks states Pages: 1 -->
<svg xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg" width="644pt" height="113pt" viewBox="0.00 0.00 643.60 113.00" preserveAspectRatio="xMidYMid meet" zoomAndPan="magnify" version="1.1" contentScriptType="application/ecmascript" contentStyleType="text/css"><defs><linearGradient id="white" x1="0%" y1="0%" x2="0%" y2="0%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/></linearGradient><linearGradient id="black" x1="0%" y1="0%" x2="0%" y2="0%"><stop offset="0%" style="stop-color:rgb(0,0,0);stop-opacity:1"/></linearGradient><linearGradient id="aquamarine" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(127,255,212);stop-opacity:1"/></linearGradient><linearGradient id="azure" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(240,255,255);stop-opacity:1"/></linearGradient><linearGradient id="blue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,0,255);stop-opacity:1"/></linearGradient><linearGradient id="blueviolet" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(138,43,226);stop-opacity:1"/></linearGradient><linearGradient id="brown" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(165,42,42);stop-opacity:1"/></linearGradient><linearGradient id="cadetblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(95,158,160);stop-opacity:1"/></linearGradient><linearGradient id="chocolate" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(210,105,30);stop-opacity:1"/></linearGradient><linearGradient id="cornflowerblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(100,149,237);stop-opacity:1"/></linearGradient><linearGradient id="crimson" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(220,20,60);stop-opacity:1"/></linearGradient><linearGradient id="cyan" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,255,255);stop-opacity:1"/></linearGradient><linearGradient id="darkgreen" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,100,0);stop-opacity:1"/></linearGradient><linearGradient id="darkorange" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,140,0);stop-opacity:1"/></linearGradient><linearGradient id="gold" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,215,0);stop-opacity:1"/></linearGradient><linearGradient id="gray" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(192,192,192);stop-opacity:1"/></linearGradient><linearGradient id="greenyellow" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(173,255,47);stop-opacity:1"/></linearGradient><linearGradient id="green" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,255,0);stop-opacity:1"/></linearGradient><linearGradient id="grey" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(192,192,192);stop-opacity:1"/></linearGradient><linearGradient id="hotpink" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,105,180);stop-opacity:1"/></linearGradient><linearGradient id="indianred" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(205,92,92);stop-opacity:1"/></linearGradient><linearGradient id="indigo" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(75,0,130);stop-opacity:1"/></linearGradient><linearGradient id="lavender" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(230,230,250);stop-opacity:1"/></linearGradient><linearGradient id="lightblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(173,216,230);stop-opacity:1"/></linearGradient><linearGradient id="lightgray" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(211,211,211);stop-opacity:1"/></linearGradient><linearGradient id="lightgrey" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(211,211,211);stop-opacity:1"/></linearGradient><linearGradient id="magenta" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,0,255);stop-opacity:1"/></linearGradient><linearGradient id="maroon" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(176,48,96);stop-opacity:1"/></linearGradient><linearGradient id="mediumblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(0,0,205);stop-opacity:1"/></linearGradient><linearGradient id="mediumpurple" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(147,112,219);stop-opacity:1"/></linearGradient><linearGradient id="orange" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,165,0);stop-opacity:1"/></linearGradient><linearGradient id="pink" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,192,203);stop-opacity:1"/></linearGradient><linearGradient id="purple" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(160,32,240);stop-opacity:1"/></linearGradient><linearGradient id="red" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,0,0);stop-opacity:1"/></linearGradient><linearGradient id="steelblue" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(70,130,180);stop-opacity:1"/></linearGradient><linearGradient id="violet" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(238,130,238);stop-opacity:1"/></linearGradient><linearGradient id="yellow" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,255,0);stop-opacity:1"/></linearGradient><linearGradient id="none" x1="0%" y1="0%" x2="100%" y2="100%"><stop offset="0%" style="stop-color:rgb(255,255,255);stop-opacity:1"/><stop offset="100%" style="stop-color:rgb(255,255,255);stop-opacity:1"/></linearGradient></defs>
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 109)"><polygon fill="white" stroke="white" points="-4,4 -4,-109 639.6,-109 639.6,4 -4,4"/><title>Tasks states</title><g id="node1" class="node"><ellipse fill="none" stroke="black" cx="103.6" cy="-79" rx="39.8775" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="103.6" cy="-79" rx="39.8775" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="103.6" y="-76.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:Verdana">PENDING</text></g><g id="node2" class="node"><ellipse fill="none" stroke="black" cx="221.6" cy="-56" rx="41.4846" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="221.6" cy="-56" rx="41.4846" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="221.6" y="-53.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:Verdana">RUNNING</text></g><g id="edge1" class="edge"><polygon fill="black" stroke="black" points="174.436,-68.7214 183.564,-63.3433 173.074,-61.8551 174.436,-68.7214" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M140.35,-71.9116C150.85,-69.8298 162.495,-67.5207 173.585,-65.3219" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="174.436,-68.7214 183.564,-63.3433 173.074,-61.8551 174.436,-68.7214" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M140.35,-71.9116C150.85,-69.8298 162.495,-67.5207 173.585,-65.3219"/></g><g id="node3" class="node"><ellipse fill="none" stroke="black" cx="338.6" cy="-18" rx="39.1741" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="338.6" cy="-18" rx="39.1741" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="338.6" y="-15.2" font-family="Times,serif" font-size="11.00" fill="green" style="font-size:10px; font-family:Verdana">SUCCESS</text></g><g id="edge2" class="edge"><polygon fill="black" stroke="black" points="298.059,-34.752 306.456,-28.2912 295.862,-28.1055 298.059,-34.752" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M255.247,-45.2124C268.178,-40.9394 283.186,-35.9803 296.82,-31.4751" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="298.059,-34.752 306.456,-28.2912 295.862,-28.1055 298.059,-34.752" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M255.247,-45.2124C268.178,-40.9394 283.186,-35.9803 296.82,-31.4751"/></g><g id="node4" class="node"><ellipse fill="none" stroke="black" cx="590.6" cy="-18" rx="38.4712" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="590.6" cy="-18" rx="38.4712" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="590.6" y="-15.2" font-family="Times,serif" font-size="11.00" fill="red" style="font-size:10px; font-family:Verdana">FAILURE</text></g><g id="edge3" class="edge"><polygon fill="black" stroke="black" points="553.052,-36.5055 561.247,-29.7894 550.653,-29.9297 553.052,-36.5055" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M262.926,-58.0652C318.829,-60.1124 422.799,-60.9658 509.6,-45 523.616,-42.422 538.524,-37.9082 551.625,-33.3006" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="553.052,-36.5055 561.247,-29.7894 550.653,-29.9297 553.052,-36.5055" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M262.926,-58.0652C318.829,-60.1124 422.799,-60.9658 509.6,-45 523.616,-42.422 538.524,-37.9082 551.625,-33.3006"/></g><g id="node5" class="node"><ellipse fill="none" stroke="black" cx="461.6" cy="-18" rx="48.2143" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="461.6" cy="-18" rx="48.2143" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="461.6" y="-15.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:Verdana">REVERTING</text></g><g id="edge4" class="edge"><polygon fill="black" stroke="black" points="403.212,-21.5001 413.212,-18 403.212,-14.5001 403.212,-21.5001" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M377.885,-18C385.947,-18 394.62,-18 403.209,-18" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="403.212,-21.5001 413.212,-18 403.212,-14.5001 403.212,-21.5001" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M377.885,-18C385.947,-18 394.62,-18 403.209,-18"/></g><g id="edge6" class="edge"><polygon fill="black" stroke="black" points="516.737,-21.1813 506.662,-24.4607 516.583,-28.1796 516.737,-21.1813" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M554.007,-24.1336C542.529,-24.6746 529.533,-24.8595 517.018,-24.6883" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="516.737,-21.1813 506.662,-24.4607 516.583,-28.1796 516.737,-21.1813" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M554.007,-24.1336C542.529,-24.6746 529.533,-24.8595 517.018,-24.6883"/></g><g id="edge7" class="edge"><polygon fill="black" stroke="black" points="543.885,-14.9952 554.007,-11.8664 544.143,-7.99992 543.885,-14.9952" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M506.662,-11.5393C518.631,-11.1724 531.619,-11.1545 543.687,-11.4856" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="543.885,-14.9952 554.007,-11.8664 544.143,-7.99992 543.885,-14.9952" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M506.662,-11.5393C518.631,-11.1724 531.619,-11.1545 543.687,-11.4856"/></g><g id="node6" class="node"><ellipse fill="none" stroke="black" cx="590.6" cy="-87" rx="45.2009" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="590.6" cy="-87" rx="45.2009" ry="18" style="fill:url(#none);stroke:black;"/><text text-anchor="middle" x="590.6" y="-84.2" font-family="Times,serif" font-size="11.00" fill="darkorange" style="font-size:10px; font-family:Verdana">REVERTED</text></g><g id="edge8" class="edge"><polygon fill="black" stroke="black" points="552.378,-70.7605 562.836,-72.4591 555.72,-64.6096 552.378,-70.7605" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M489.845,-32.8022C508.553,-42.9667 533.576,-56.5617 553.879,-67.5927" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="552.378,-70.7605 562.836,-72.4591 555.72,-64.6096 552.378,-70.7605" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M489.845,-32.8022C508.553,-42.9667 533.576,-56.5617 553.879,-67.5927"/></g><g id="edge5" class="edge"><polygon fill="black" stroke="black" points="153.295,-78.4014 143.126,-81.377 152.931,-85.3919 153.295,-78.4014" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M545.297,-87.1991C470.091,-87.3957 312.648,-87.195 179.6,-83 171.076,-82.7312 162.011,-82.3387 153.24,-81.9032" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="153.295,-78.4014 143.126,-81.377 152.931,-85.3919 153.295,-78.4014" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" d="M545.297,-87.1991C470.091,-87.3957 312.648,-87.195 179.6,-83 171.076,-82.7312 162.011,-82.3387 153.24,-81.9032"/></g><g id="node7" class="node"><ellipse fill="black" stroke="black" cx="23.6" cy="-79" rx="3.6" ry="3.6" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="black" stroke="black" cx="23.6" cy="-79" rx="3.6" ry="3.6" style="fill:url(#black);stroke:black;"/><text text-anchor="middle" x="10" y="-66.6" font-family="Times,serif" font-size="11.00" fill="green" style="font-size:10px; font-family:Verdana">start</text></g><g id="edge9" class="edge"><polygon fill="black" stroke="black" points="53.5965,-82.5001 63.5964,-79 53.5964,-75.5001 53.5965,-82.5001" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" stroke-dasharray="1,5" d="M27.5624,-79C32.3405,-79 42.3529,-79 53.5516,-79" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="53.5965,-82.5001 63.5964,-79 53.5964,-75.5001 53.5965,-82.5001" style="fill:url(#black);stroke:black;"/><path fill="none" stroke="black" stroke-dasharray="1,5" d="M27.5624,-79C32.3405,-79 42.3529,-79 53.5516,-79"/></g></g>
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 109)"><polygon fill="white" stroke="none" points="-4,4 -4,-109 639.6,-109 639.6,4 -4,4"/><title>Tasks states</title><g id="node1" class="node"><ellipse fill="none" stroke="black" cx="103.6" cy="-79" rx="39.8775" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="103.6" cy="-79" rx="39.8775" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="103.6" y="-76.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:sans-serif;">PENDING</text></g><g id="node2" class="node"><ellipse fill="none" stroke="black" cx="221.6" cy="-56" rx="41.4846" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="221.6" cy="-56" rx="41.4846" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="221.6" y="-53.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:sans-serif;">RUNNING</text></g><g id="edge1" class="edge"><polygon fill="black" stroke="black" points="174.436,-68.7214 183.564,-63.3433 173.074,-61.8551 174.436,-68.7214" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M140.35,-71.9116C150.85,-69.8298 162.495,-67.5207 173.585,-65.3219" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="174.436,-68.7214 183.564,-63.3433 173.074,-61.8551 174.436,-68.7214" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M140.35,-71.9116C150.85,-69.8298 162.495,-67.5207 173.585,-65.3219"/></g><g id="node3" class="node"><ellipse fill="none" stroke="black" cx="338.6" cy="-18" rx="39.1741" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="338.6" cy="-18" rx="39.1741" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="338.6" y="-15.2" font-family="Times,serif" font-size="11.00" fill="green" style="font-size:10px; font-family:sans-serif;">SUCCESS</text></g><g id="edge2" class="edge"><polygon fill="black" stroke="black" points="298.059,-34.752 306.456,-28.2912 295.862,-28.1055 298.059,-34.752" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M255.247,-45.2124C268.178,-40.9394 283.186,-35.9803 296.82,-31.4751" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="298.059,-34.752 306.456,-28.2912 295.862,-28.1055 298.059,-34.752" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M255.247,-45.2124C268.178,-40.9394 283.186,-35.9803 296.82,-31.4751"/></g><g id="node4" class="node"><ellipse fill="none" stroke="black" cx="590.6" cy="-18" rx="38.4712" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="590.6" cy="-18" rx="38.4712" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="590.6" y="-15.2" font-family="Times,serif" font-size="11.00" fill="red" style="font-size:10px; font-family:sans-serif;">FAILURE</text></g><g id="edge3" class="edge"><polygon fill="black" stroke="black" points="553.052,-36.5055 561.247,-29.7894 550.653,-29.9297 553.052,-36.5055" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M262.926,-58.0652C318.829,-60.1124 422.799,-60.9658 509.6,-45 523.616,-42.422 538.524,-37.9082 551.625,-33.3006" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="553.052,-36.5055 561.247,-29.7894 550.653,-29.9297 553.052,-36.5055" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M262.926,-58.0652C318.829,-60.1124 422.799,-60.9658 509.6,-45 523.616,-42.422 538.524,-37.9082 551.625,-33.3006"/></g><g id="node5" class="node"><ellipse fill="none" stroke="black" cx="461.6" cy="-18" rx="48.2143" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="461.6" cy="-18" rx="48.2143" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="461.6" y="-15.2" font-family="Times,serif" font-size="11.00" style="font-size:10px; font-family:sans-serif;">REVERTING</text></g><g id="edge4" class="edge"><polygon fill="black" stroke="black" points="403.212,-21.5001 413.212,-18 403.212,-14.5001 403.212,-21.5001" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M377.885,-18C385.947,-18 394.62,-18 403.209,-18" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="403.212,-21.5001 413.212,-18 403.212,-14.5001 403.212,-21.5001" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M377.885,-18C385.947,-18 394.62,-18 403.209,-18"/></g><g id="edge6" class="edge"><polygon fill="black" stroke="black" points="516.737,-21.1813 506.662,-24.4607 516.583,-28.1796 516.737,-21.1813" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M554.007,-24.1336C542.529,-24.6746 529.533,-24.8595 517.018,-24.6883" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="516.737,-21.1813 506.662,-24.4607 516.583,-28.1796 516.737,-21.1813" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M554.007,-24.1336C542.529,-24.6746 529.533,-24.8595 517.018,-24.6883"/></g><g id="edge7" class="edge"><polygon fill="black" stroke="black" points="543.885,-14.9952 554.007,-11.8664 544.143,-7.99992 543.885,-14.9952" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M506.662,-11.5393C518.631,-11.1724 531.619,-11.1545 543.687,-11.4856" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="543.885,-14.9952 554.007,-11.8664 544.143,-7.99992 543.885,-14.9952" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M506.662,-11.5393C518.631,-11.1724 531.619,-11.1545 543.687,-11.4856"/></g><g id="node6" class="node"><ellipse fill="none" stroke="black" cx="590.6" cy="-87" rx="45.2009" ry="18" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="none" stroke="black" cx="590.6" cy="-87" rx="45.2009" ry="18" style="fill: url(#none);stroke: black;"/><text text-anchor="middle" x="590.6" y="-84.2" font-family="Times,serif" font-size="11.00" fill="darkorange" style="font-size:10px; font-family:sans-serif;">REVERTED</text></g><g id="edge8" class="edge"><polygon fill="black" stroke="black" points="552.378,-70.7605 562.836,-72.4591 555.72,-64.6096 552.378,-70.7605" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M489.845,-32.8022C508.553,-42.9667 533.576,-56.5617 553.879,-67.5927" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="552.378,-70.7605 562.836,-72.4591 555.72,-64.6096 552.378,-70.7605" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M489.845,-32.8022C508.553,-42.9667 533.576,-56.5617 553.879,-67.5927"/></g><g id="edge5" class="edge"><polygon fill="black" stroke="black" points="153.295,-78.4014 143.126,-81.377 152.931,-85.3919 153.295,-78.4014" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" d="M545.297,-87.1991C470.091,-87.3957 312.648,-87.195 179.6,-83 171.076,-82.7312 162.011,-82.3387 153.24,-81.9032" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="153.295,-78.4014 143.126,-81.377 152.931,-85.3919 153.295,-78.4014" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" d="M545.297,-87.1991C470.091,-87.3957 312.648,-87.195 179.6,-83 171.076,-82.7312 162.011,-82.3387 153.24,-81.9032"/></g><g id="node7" class="node"><ellipse fill="black" stroke="black" cx="23.6" cy="-79" rx="3.6" ry="3.6" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><ellipse fill="black" stroke="black" cx="23.6" cy="-79" rx="3.6" ry="3.6" style="fill: url(#black);stroke: black;"/><text text-anchor="middle" x="10" y="-66.6" font-family="Times,serif" font-size="11.00" fill="green" style="font-size:10px; font-family:sans-serif;">start</text></g><g id="edge9" class="edge"><polygon fill="black" stroke="black" points="53.5965,-82.5001 63.5964,-79 53.5964,-75.5001 53.5965,-82.5001" style="fill: black; stroke: none; fill-opacity:0.3" transform="translate(3,3)"/><path fill="none" stroke="black" stroke-dasharray="1,5" d="M27.5624,-79C32.3405,-79 42.3529,-79 53.5516,-79" style="fill: none; stroke: black; stroke-opacity:0.3" transform="translate(3,3)"/><polygon fill="black" stroke="black" points="53.5965,-82.5001 63.5964,-79 53.5964,-75.5001 53.5965,-82.5001" style="fill: url(#black);stroke: black;"/><path fill="none" stroke="black" stroke-dasharray="1,5" d="M27.5624,-79C32.3405,-79 42.3529,-79 53.5516,-79"/></g></g>
</svg>
diff --git a/doc/source/utils.rst b/doc/source/utils.rst
index 6949ccf..ac0dd5c 100644
--- a/doc/source/utils.rst
+++ b/doc/source/utils.rst
@@ -33,11 +33,6 @@ Kombu
.. automodule:: taskflow.utils.kombu_utils
-Locks
-~~~~~
-
-.. automodule:: taskflow.utils.lock_utils
-
Miscellaneous
~~~~~~~~~~~~~
diff --git a/requirements.txt b/requirements.txt
index 43efdec..a2ce906 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,7 +17,7 @@ six>=1.9.0
enum34
# For reader/writer + interprocess locks.
-fasteners>=0.5 # Apache-2.0
+fasteners>=0.7 # Apache-2.0
# Very nice graph library
networkx>=1.8
@@ -26,7 +26,7 @@ networkx>=1.8
contextlib2>=0.4.0 # PSF License
# Used for backend storage engine loading.
-stevedore>=1.3.0 # Apache-2.0
+stevedore>=1.5.0 # Apache-2.0
# Backport for concurrent.futures which exists in 3.2+
futures>=3.0
@@ -35,7 +35,7 @@ futures>=3.0
jsonschema>=2.0.0,<3.0.0,!=2.5.0
# For common utilities
-oslo.utils>=1.4.0 # Apache-2.0
+oslo.utils>=1.6.0 # Apache-2.0
oslo.serialization>=1.4.0 # Apache-2.0
# For lru caches and such
diff --git a/taskflow/conductors/base.py b/taskflow/conductors/base.py
index 7a6b8ce..6e46fff 100644
--- a/taskflow/conductors/base.py
+++ b/taskflow/conductors/base.py
@@ -15,11 +15,11 @@
import abc
import threading
+import fasteners
import six
from taskflow import engines
from taskflow import exceptions as excp
-from taskflow.utils import lock_utils
@six.add_metaclass(abc.ABCMeta)
@@ -109,13 +109,13 @@ class Conductor(object):
# listener factories over the jobboard
return []
- @lock_utils.locked
+ @fasteners.locked
def connect(self):
"""Ensures the jobboard is connected (noop if it is already)."""
if not self._jobboard.connected:
self._jobboard.connect()
- @lock_utils.locked
+ @fasteners.locked
def close(self):
"""Closes the contained jobboard, disallowing further use."""
self._jobboard.close()
diff --git a/taskflow/engines/action_engine/actions/base.py b/taskflow/engines/action_engine/actions/base.py
index 869ef22..369a6c6 100644
--- a/taskflow/engines/action_engine/actions/base.py
+++ b/taskflow/engines/action_engine/actions/base.py
@@ -35,7 +35,3 @@ class Action(object):
def __init__(self, storage, notifier):
self._storage = storage
self._notifier = notifier
-
- @abc.abstractmethod
- def handles(self, atom):
- """Checks if this action handles the provided atom."""
diff --git a/taskflow/engines/action_engine/actions/retry.py b/taskflow/engines/action_engine/actions/retry.py
index f69d5a5..c8cad50 100644
--- a/taskflow/engines/action_engine/actions/retry.py
+++ b/taskflow/engines/action_engine/actions/retry.py
@@ -48,10 +48,6 @@ class RetryAction(base.Action):
super(RetryAction, self).__init__(storage, notifier)
self._executor = futures.SynchronousExecutor()
- @staticmethod
- def handles(atom):
- return isinstance(atom, retry_atom.Retry)
-
def _get_retry_args(self, retry, addons=None):
arguments = self._storage.fetch_mapped_args(
retry.rebind,
diff --git a/taskflow/engines/action_engine/actions/task.py b/taskflow/engines/action_engine/actions/task.py
index 2a11bf8..ab4b50d 100644
--- a/taskflow/engines/action_engine/actions/task.py
+++ b/taskflow/engines/action_engine/actions/task.py
@@ -32,10 +32,6 @@ class TaskAction(base.Action):
super(TaskAction, self).__init__(storage, notifier)
self._task_executor = task_executor
- @staticmethod
- def handles(atom):
- return isinstance(atom, task_atom.BaseTask)
-
def _is_identity_transition(self, old_state, state, task, progress):
if state in base.SAVE_RESULT_STATES:
# saving result is never identity transition
diff --git a/taskflow/engines/action_engine/analyzer.py b/taskflow/engines/action_engine/analyzer.py
index a8f20c0..bef7b8b 100644
--- a/taskflow/engines/action_engine/analyzer.py
+++ b/taskflow/engines/action_engine/analyzer.py
@@ -34,6 +34,7 @@ class Analyzer(object):
def __init__(self, runtime):
self._storage = runtime.storage
self._execution_graph = runtime.compilation.execution_graph
+ self._check_atom_transition = runtime.check_atom_transition
def get_next_nodes(self, node=None):
if node is None:
@@ -93,37 +94,37 @@ class Analyzer(object):
available_nodes.append(node)
return available_nodes
- def _is_ready_for_execute(self, task):
- """Checks if task is ready to be executed."""
- state = self.get_state(task)
- intention = self._storage.get_atom_intention(task.name)
- transition = st.check_task_transition(state, st.RUNNING)
+ def _is_ready_for_execute(self, atom):
+ """Checks if atom is ready to be executed."""
+ state = self.get_state(atom)
+ intention = self._storage.get_atom_intention(atom.name)
+ transition = self._check_atom_transition(atom, state, st.RUNNING)
if not transition or intention != st.EXECUTE:
return False
- task_names = []
- for prev_task in self._execution_graph.predecessors(task):
- task_names.append(prev_task.name)
+ atom_names = []
+ for prev_atom in self._execution_graph.predecessors(atom):
+ atom_names.append(prev_atom.name)
- task_states = self._storage.get_atoms_states(task_names)
+ atom_states = self._storage.get_atoms_states(atom_names)
return all(state == st.SUCCESS and intention == st.EXECUTE
- for state, intention in six.itervalues(task_states))
+ for state, intention in six.itervalues(atom_states))
- def _is_ready_for_revert(self, task):
- """Checks if task is ready to be reverted."""
- state = self.get_state(task)
- intention = self._storage.get_atom_intention(task.name)
- transition = st.check_task_transition(state, st.REVERTING)
+ def _is_ready_for_revert(self, atom):
+ """Checks if atom is ready to be reverted."""
+ state = self.get_state(atom)
+ intention = self._storage.get_atom_intention(atom.name)
+ transition = self._check_atom_transition(atom, state, st.REVERTING)
if not transition or intention not in (st.REVERT, st.RETRY):
return False
- task_names = []
- for prev_task in self._execution_graph.successors(task):
- task_names.append(prev_task.name)
+ atom_names = []
+ for prev_atom in self._execution_graph.successors(atom):
+ atom_names.append(prev_atom.name)
- task_states = self._storage.get_atoms_states(task_names)
+ atom_states = self._storage.get_atoms_states(atom_names)
return all(state in (st.PENDING, st.REVERTED)
- for state, intention in six.itervalues(task_states))
+ for state, intention in six.itervalues(atom_states))
def iterate_subgraph(self, atom):
"""Iterates a subgraph connected to given atom."""
@@ -148,10 +149,10 @@ class Analyzer(object):
return self._execution_graph.node[atom].get('retry')
def is_success(self):
- for node in self._execution_graph.nodes_iter():
- if self.get_state(node) != st.SUCCESS:
+ for atom in self.iterate_all_nodes():
+ if self.get_state(atom) != st.SUCCESS:
return False
return True
- def get_state(self, node):
- return self._storage.get_atom_state(node.name)
+ def get_state(self, atom):
+ return self._storage.get_atom_state(atom.name)
diff --git a/taskflow/engines/action_engine/compiler.py b/taskflow/engines/action_engine/compiler.py
index ff86de2..dc6c24e 100644
--- a/taskflow/engines/action_engine/compiler.py
+++ b/taskflow/engines/action_engine/compiler.py
@@ -17,13 +17,14 @@
import collections
import threading
+import fasteners
+
from taskflow import exceptions as exc
from taskflow import flow
from taskflow import logging
from taskflow import task
from taskflow.types import graph as gr
from taskflow.types import tree as tr
-from taskflow.utils import lock_utils
from taskflow.utils import misc
LOG = logging.getLogger(__name__)
@@ -423,7 +424,7 @@ class PatternCompiler(object):
# Indent it so that it's slightly offset from the above line.
LOG.blather(" %s", line)
- @lock_utils.locked
+ @fasteners.locked
def compile(self):
"""Compiles the contained item into a compiled equivalent."""
if self._compilation is None:
diff --git a/taskflow/engines/action_engine/engine.py b/taskflow/engines/action_engine/engine.py
index 3dda912..df8d1d3 100644
--- a/taskflow/engines/action_engine/engine.py
+++ b/taskflow/engines/action_engine/engine.py
@@ -19,6 +19,7 @@ import contextlib
import threading
from concurrent import futures
+import fasteners
import networkx as nx
from oslo_utils import excutils
from oslo_utils import strutils
@@ -33,7 +34,6 @@ from taskflow import logging
from taskflow import states
from taskflow import storage
from taskflow.types import failure
-from taskflow.utils import lock_utils
from taskflow.utils import misc
LOG = logging.getLogger(__name__)
@@ -133,7 +133,7 @@ class ActionEngine(base.Engine):
scope_fetcher=_scope_fetcher)
def run(self):
- with lock_utils.try_lock(self._lock) as was_locked:
+ with fasteners.try_lock(self._lock) as was_locked:
if not was_locked:
raise exc.ExecutionFailure("Engine currently locked, please"
" try again later")
@@ -222,7 +222,7 @@ class ActionEngine(base.Engine):
node.inject,
transient=transient)
- @lock_utils.locked
+ @fasteners.locked
def validate(self):
self._check('validate', True, True)
# At this point we can check to ensure all dependencies are either
@@ -266,7 +266,7 @@ class ActionEngine(base.Engine):
sorted(missing),
cause=last_cause)
- @lock_utils.locked
+ @fasteners.locked
def prepare(self):
self._check('prepare', True, False)
if not self._storage_ensured:
@@ -286,7 +286,7 @@ class ActionEngine(base.Engine):
def _compiler(self):
return self._compiler_factory(self._flow)
- @lock_utils.locked
+ @fasteners.locked
def compile(self):
if self._compiled:
return
@@ -295,6 +295,7 @@ class ActionEngine(base.Engine):
self.storage,
self.atom_notifier,
self._task_executor)
+ self._runtime.compile()
self._compiled = True
diff --git a/taskflow/engines/action_engine/runner.py b/taskflow/engines/action_engine/runner.py
index d50de15..8d637c1 100644
--- a/taskflow/engines/action_engine/runner.py
+++ b/taskflow/engines/action_engine/runner.py
@@ -51,38 +51,48 @@ class _MachineMemory(object):
self.done = set()
-class _MachineBuilder(object):
- """State machine *builder* that the runner uses.
-
- NOTE(harlowja): the machine states that this build will for are::
-
- +--------------+------------------+------------+----------+---------+
- Start | Event | End | On Enter | On Exit
- +--------------+------------------+------------+----------+---------+
- ANALYZING | completed | GAME_OVER | |
- ANALYZING | schedule_next | SCHEDULING | |
- ANALYZING | wait_finished | WAITING | |
- FAILURE[$] | | | |
- GAME_OVER | failed | FAILURE | |
- GAME_OVER | reverted | REVERTED | |
- GAME_OVER | success | SUCCESS | |
- GAME_OVER | suspended | SUSPENDED | |
- RESUMING | schedule_next | SCHEDULING | |
- REVERTED[$] | | | |
- SCHEDULING | wait_finished | WAITING | |
- SUCCESS[$] | | | |
- SUSPENDED[$] | | | |
- UNDEFINED[^] | start | RESUMING | |
- WAITING | examine_finished | ANALYZING | |
- +--------------+------------------+------------+----------+---------+
+class Runner(object):
+ """State machine *builder* + *runner* that powers the engine components.
+
+ NOTE(harlowja): the machine (states and events that will trigger
+ transitions) that this builds is represented by the following
+ table::
+
+ +--------------+------------------+------------+----------+---------+
+ Start | Event | End | On Enter | On Exit
+ +--------------+------------------+------------+----------+---------+
+ ANALYZING | completed | GAME_OVER | |
+ ANALYZING | schedule_next | SCHEDULING | |
+ ANALYZING | wait_finished | WAITING | |
+ FAILURE[$] | | | |
+ GAME_OVER | failed | FAILURE | |
+ GAME_OVER | reverted | REVERTED | |
+ GAME_OVER | success | SUCCESS | |
+ GAME_OVER | suspended | SUSPENDED | |
+ RESUMING | schedule_next | SCHEDULING | |
+ REVERTED[$] | | | |
+ SCHEDULING | wait_finished | WAITING | |
+ SUCCESS[$] | | | |
+ SUSPENDED[$] | | | |
+ UNDEFINED[^] | start | RESUMING | |
+ WAITING | examine_finished | ANALYZING | |
+ +--------------+------------------+------------+----------+---------+
Between any of these yielded states (minus ``GAME_OVER`` and ``UNDEFINED``)
if the engine has been suspended or the engine has failed (due to a
non-resolveable task failure or scheduling failure) the machine will stop
executing new tasks (currently running tasks will be allowed to complete)
and this machines run loop will be broken.
+
+ NOTE(harlowja): If the runtimes scheduler component is able to schedule
+ tasks in parallel, this enables parallel running and/or reversion.
"""
+ # Informational states this action yields while running, not useful to
+ # have the engine record but useful to provide to end-users when doing
+ # execution iterations.
+ ignorable_states = (st.SCHEDULING, st.WAITING, st.RESUMING, st.ANALYZING)
+
def __init__(self, runtime, waiter):
self._analyzer = runtime.analyzer
self._completer = runtime.completer
@@ -91,9 +101,12 @@ class _MachineBuilder(object):
self._waiter = waiter
def runnable(self):
+ """Checks if the storage says the flow is still runnable/running."""
return self._storage.get_flow_state() == st.RUNNING
def build(self, timeout=None):
+ """Builds a state-machine (that can be/is used during running)."""
+
memory = _MachineMemory()
if timeout is None:
timeout = _WAITING_TIMEOUT
@@ -244,38 +257,9 @@ class _MachineBuilder(object):
m.freeze()
return (m, memory)
-
-class Runner(object):
- """Runner that iterates while executing nodes using the given runtime.
-
- This runner acts as the action engine run loop/state-machine, it resumes
- the workflow, schedules all task it can for execution using the runtimes
- scheduler and analyzer components, and than waits on returned futures and
- then activates the runtimes completion component to finish up those tasks
- and so on...
-
- NOTE(harlowja): If the runtimes scheduler component is able to schedule
- tasks in parallel, this enables parallel running and/or reversion.
- """
-
- # Informational states this action yields while running, not useful to
- # have the engine record but useful to provide to end-users when doing
- # execution iterations.
- ignorable_states = (st.SCHEDULING, st.WAITING, st.RESUMING, st.ANALYZING)
-
- def __init__(self, runtime, waiter):
- self._builder = _MachineBuilder(runtime, waiter)
-
- @property
- def builder(self):
- return self._builder
-
- def runnable(self):
- return self._builder.runnable()
-
def run_iter(self, timeout=None):
- """Runs the nodes using a built state machine."""
- machine, memory = self.builder.build(timeout=timeout)
+ """Runs iteratively using a locally built state machine."""
+ machine, memory = self.build(timeout=timeout)
for (_prior_state, new_state) in machine.run_iter(_START):
# NOTE(harlowja): skip over meta-states.
if new_state not in _META_STATES:
diff --git a/taskflow/engines/action_engine/runtime.py b/taskflow/engines/action_engine/runtime.py
index fc16fd9..061cca4 100644
--- a/taskflow/engines/action_engine/runtime.py
+++ b/taskflow/engines/action_engine/runtime.py
@@ -14,6 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
+import functools
+
from taskflow.engines.action_engine.actions import retry as ra
from taskflow.engines.action_engine.actions import task as ta
from taskflow.engines.action_engine import analyzer as an
@@ -22,6 +24,7 @@ from taskflow.engines.action_engine import runner as ru
from taskflow.engines.action_engine import scheduler as sched
from taskflow.engines.action_engine import scopes as sc
from taskflow import states as st
+from taskflow import task
from taskflow.utils import misc
@@ -38,7 +41,37 @@ class Runtime(object):
self._task_executor = task_executor
self._storage = storage
self._compilation = compilation
- self._walkers_to_names = {}
+ self._atom_cache = {}
+
+ def compile(self):
+ # Build out a cache of commonly used item that are associated
+ # with the contained atoms (by name), and are useful to have for
+ # quick lookup on...
+ change_state_handlers = {
+ 'task': functools.partial(self.task_action.change_state,
+ progress=0.0),
+ 'retry': self.retry_action.change_state,
+ }
+ schedulers = {
+ 'retry': self.retry_scheduler,
+ 'task': self.task_scheduler,
+ }
+ for atom in self.analyzer.iterate_all_nodes():
+ metadata = {}
+ walker = sc.ScopeWalker(self.compilation, atom, names_only=True)
+ if isinstance(atom, task.BaseTask):
+ check_transition_handler = st.check_task_transition
+ change_state_handler = change_state_handlers['task']
+ scheduler = schedulers['task']
+ else:
+ check_transition_handler = st.check_retry_transition
+ change_state_handler = change_state_handlers['retry']
+ scheduler = schedulers['retry']
+ metadata['scope_walker'] = walker
+ metadata['check_transition_handler'] = check_transition_handler
+ metadata['change_state_handler'] = change_state_handler
+ metadata['scheduler'] = scheduler
+ self._atom_cache[atom.name] = metadata
@property
def compilation(self):
@@ -65,6 +98,14 @@ class Runtime(object):
return sched.Scheduler(self)
@misc.cachedproperty
+ def task_scheduler(self):
+ return sched.TaskScheduler(self)
+
+ @misc.cachedproperty
+ def retry_scheduler(self):
+ return sched.RetryScheduler(self)
+
+ @misc.cachedproperty
def retry_action(self):
return ra.RetryAction(self._storage,
self._atom_notifier)
@@ -75,56 +116,60 @@ class Runtime(object):
self._atom_notifier,
self._task_executor)
+ def check_atom_transition(self, atom, current_state, target_state):
+ """Checks if the atom can transition to the provided target state."""
+ # This does not check if the name exists (since this is only used
+ # internally to the engine, and is not exposed to atoms that will
+ # not exist and therefore doesn't need to handle that case).
+ metadata = self._atom_cache[atom.name]
+ check_transition_handler = metadata['check_transition_handler']
+ return check_transition_handler(current_state, target_state)
+
+ def fetch_scheduler(self, atom):
+ """Fetches the cached specific scheduler for the given atom."""
+ # This does not check if the name exists (since this is only used
+ # internally to the engine, and is not exposed to atoms that will
+ # not exist and therefore doesn't need to handle that case).
+ metadata = self._atom_cache[atom.name]
+ return metadata['scheduler']
+
def fetch_scopes_for(self, atom_name):
"""Fetches a walker of the visible scopes for the given atom."""
try:
- return self._walkers_to_names[atom_name]
+ metadata = self._atom_cache[atom_name]
except KeyError:
- atom = None
- for node in self.analyzer.iterate_all_nodes():
- if node.name == atom_name:
- atom = node
- break
- if atom is not None:
- walker = sc.ScopeWalker(self.compilation, atom,
- names_only=True)
- self._walkers_to_names[atom_name] = walker
- else:
- walker = None
- return walker
+ # This signals to the caller that there is no walker for whatever
+ # atom name was given that doesn't really have any associated atom
+ # known to be named with that name; this is done since the storage
+ # layer will call into this layer to fetch a scope for a named
+ # atom and users can provide random names that do not actually
+ # exist...
+ return None
+ else:
+ return metadata['scope_walker']
# Various helper methods used by the runtime components; not for public
# consumption...
- def reset_nodes(self, nodes, state=st.PENDING, intention=st.EXECUTE):
+ def reset_nodes(self, atoms, state=st.PENDING, intention=st.EXECUTE):
tweaked = []
- node_state_handlers = [
- (self.task_action, {'progress': 0.0}),
- (self.retry_action, {}),
- ]
- for node in nodes:
+ for atom in atoms:
+ metadata = self._atom_cache[atom.name]
if state or intention:
- tweaked.append((node, state, intention))
+ tweaked.append((atom, state, intention))
if state:
- handled = False
- for h, kwargs in node_state_handlers:
- if h.handles(node):
- h.change_state(node, state, **kwargs)
- handled = True
- break
- if not handled:
- raise TypeError("Unknown how to reset state of"
- " node '%s' (%s)" % (node, type(node)))
+ change_state_handler = metadata['change_state_handler']
+ change_state_handler(atom, state)
if intention:
- self.storage.set_atom_intention(node.name, intention)
+ self.storage.set_atom_intention(atom.name, intention)
return tweaked
def reset_all(self, state=st.PENDING, intention=st.EXECUTE):
return self.reset_nodes(self.analyzer.iterate_all_nodes(),
state=state, intention=intention)
- def reset_subgraph(self, node, state=st.PENDING, intention=st.EXECUTE):
- return self.reset_nodes(self.analyzer.iterate_subgraph(node),
+ def reset_subgraph(self, atom, state=st.PENDING, intention=st.EXECUTE):
+ return self.reset_nodes(self.analyzer.iterate_subgraph(atom),
state=state, intention=intention)
def retry_subflow(self, retry):
diff --git a/taskflow/engines/action_engine/scheduler.py b/taskflow/engines/action_engine/scheduler.py
index 2022183..4ab0b0e 100644
--- a/taskflow/engines/action_engine/scheduler.py
+++ b/taskflow/engines/action_engine/scheduler.py
@@ -17,22 +17,18 @@
import weakref
from taskflow import exceptions as excp
-from taskflow import retry as retry_atom
from taskflow import states as st
-from taskflow import task as task_atom
from taskflow.types import failure
-class _RetryScheduler(object):
+class RetryScheduler(object):
+ """Schedules retry atoms."""
+
def __init__(self, runtime):
self._runtime = weakref.proxy(runtime)
self._retry_action = runtime.retry_action
self._storage = runtime.storage
- @staticmethod
- def handles(atom):
- return isinstance(atom, retry_atom.Retry)
-
def schedule(self, retry):
"""Schedules the given retry atom for *future* completion.
@@ -53,15 +49,13 @@ class _RetryScheduler(object):
" intention: %s" % intention)
-class _TaskScheduler(object):
+class TaskScheduler(object):
+ """Schedules task atoms."""
+
def __init__(self, runtime):
self._storage = runtime.storage
self._task_action = runtime.task_action
- @staticmethod
- def handles(atom):
- return isinstance(atom, task_atom.BaseTask)
-
def schedule(self, task):
"""Schedules the given task atom for *future* completion.
@@ -79,39 +73,28 @@ class _TaskScheduler(object):
class Scheduler(object):
- """Schedules atoms using actions to schedule."""
+ """Safely schedules atoms using a runtime ``fetch_scheduler`` routine."""
def __init__(self, runtime):
- self._schedulers = [
- _RetryScheduler(runtime),
- _TaskScheduler(runtime),
- ]
-
- def _schedule_node(self, node):
- """Schedule a single node for execution."""
- for sched in self._schedulers:
- if sched.handles(node):
- return sched.schedule(node)
- else:
- raise TypeError("Unknown how to schedule '%s' (%s)"
- % (node, type(node)))
+ self._fetch_scheduler = runtime.fetch_scheduler
- def schedule(self, nodes):
- """Schedules the provided nodes for *future* completion.
+ def schedule(self, atoms):
+ """Schedules the provided atoms for *future* completion.
- This method should schedule a future for each node provided and return
+ This method should schedule a future for each atom provided and return
a set of those futures to be waited on (or used for other similar
purposes). It should also return any failure objects that represented
scheduling failures that may have occurred during this scheduling
process.
"""
futures = set()
- for node in nodes:
+ for atom in atoms:
+ scheduler = self._fetch_scheduler(atom)
try:
- futures.add(self._schedule_node(node))
+ futures.add(scheduler.schedule(atom))
except Exception:
# Immediately stop scheduling future work so that we can
- # exit execution early (rather than later) if a single task
+ # exit execution early (rather than later) if a single atom
# fails to schedule correctly.
return (futures, [failure.Failure()])
return (futures, [])
diff --git a/taskflow/engines/worker_based/protocol.py b/taskflow/engines/worker_based/protocol.py
index 867f536..cbb61eb 100644
--- a/taskflow/engines/worker_based/protocol.py
+++ b/taskflow/engines/worker_based/protocol.py
@@ -19,6 +19,7 @@ import collections
import threading
from concurrent import futures
+import fasteners
from oslo_utils import reflection
from oslo_utils import timeutils
import six
@@ -28,7 +29,6 @@ from taskflow import exceptions as excp
from taskflow import logging
from taskflow.types import failure as ft
from taskflow.types import timing as tt
-from taskflow.utils import lock_utils
from taskflow.utils import schema_utils as su
# NOTE(skudriashev): This is protocol states and events, which are not
@@ -336,7 +336,7 @@ class Request(Message):
new_state, exc_info=True)
return moved
- @lock_utils.locked
+ @fasteners.locked
def transition(self, new_state):
"""Transitions the request to a new state.
diff --git a/taskflow/examples/job_board_no_test.py b/taskflow/examples/job_board_no_test.py
deleted file mode 100644
index d37c96a..0000000
--- a/taskflow/examples/job_board_no_test.py
+++ /dev/null
@@ -1,171 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# Copyright © 2013 eNovance <licensing@enovance.com>
-#
-# Authors: Dan Krause <dan@dankrause.net>
-# Cyril Roelandt <cyril.roelandt@enovance.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This example shows how to use the job board feature.
-#
-# Let's start by creating some jobs:
-# $ python job_board_no_test.py create my-board my-job '{}'
-# $ python job_board_no_test.py create my-board my-job '{"foo": "bar"}'
-# $ python job_board_no_test.py create my-board my-job '{"foo": "baz"}'
-# $ python job_board_no_test.py create my-board my-job '{"foo": "barbaz"}'
-#
-# Make sure they were registered:
-# $ python job_board_no_test.py list my-board
-# 7277181a-1f83-473d-8233-f361615bae9e - {}
-# 84a396e8-d02e-450d-8566-d93cb68550c0 - {u'foo': u'bar'}
-# 4d355d6a-2c72-44a2-a558-19ae52e8ae2c - {u'foo': u'baz'}
-# cd9aae2c-fd64-416d-8ba0-426fa8e3d59c - {u'foo': u'barbaz'}
-#
-# Perform one job:
-# $ python job_board_no_test.py consume my-board \
-# 84a396e8-d02e-450d-8566-d93cb68550c0
-# Performing job 84a396e8-d02e-450d-8566-d93cb68550c0 with args \
-# {u'foo': u'bar'}
-# $ python job_board_no_test.py list my-board
-# 7277181a-1f83-473d-8233-f361615bae9e - {}
-# 4d355d6a-2c72-44a2-a558-19ae52e8ae2c - {u'foo': u'baz'}
-# cd9aae2c-fd64-416d-8ba0-426fa8e3d59c - {u'foo': u'barbaz'}
-#
-# Delete a job:
-# $ python job_board_no_test.py delete my-board \
-# cd9aae2c-fd64-416d-8ba0-426fa8e3d59c
-# $ python job_board_no_test.py list my-board
-# 7277181a-1f83-473d-8233-f361615bae9e - {}
-# 4d355d6a-2c72-44a2-a558-19ae52e8ae2c - {u'foo': u'baz'}
-#
-# Delete all the remaining jobs
-# $ python job_board_no_test.py clear my-board
-# $ python job_board_no_test.py list my-board
-# $
-
-import argparse
-import contextlib
-import json
-import os
-import sys
-import tempfile
-
-import taskflow.jobs.backends as job_backends
-from taskflow.persistence import logbook
-
-import example_utils # noqa
-
-
-@contextlib.contextmanager
-def jobboard(*args, **kwargs):
- jb = job_backends.fetch(*args, **kwargs)
- jb.connect()
- yield jb
- jb.close()
-
-
-conf = {
- 'board': 'zookeeper',
- 'hosts': ['127.0.0.1:2181']
-}
-
-
-def consume_job(args):
- def perform_job(job):
- print("Performing job %s with args %s" % (job.uuid, job.details))
-
- with jobboard(args.board_name, conf) as jb:
- for job in jb.iterjobs(ensure_fresh=True):
- if job.uuid == args.job_uuid:
- jb.claim(job, "test-client")
- perform_job(job)
- jb.consume(job, "test-client")
-
-
-def clear_jobs(args):
- with jobboard(args.board_name, conf) as jb:
- for job in jb.iterjobs(ensure_fresh=True):
- jb.claim(job, "test-client")
- jb.consume(job, "test-client")
-
-
-def create_job(args):
- store = json.loads(args.details)
- book = logbook.LogBook(args.job_name)
- if example_utils.SQLALCHEMY_AVAILABLE:
- persist_path = os.path.join(tempfile.gettempdir(), "persisting.db")
- backend_uri = "sqlite:///%s" % (persist_path)
- else:
- persist_path = os.path.join(tempfile.gettempdir(), "persisting")
- backend_uri = "file:///%s" % (persist_path)
- with example_utils.get_backend(backend_uri) as backend:
- backend.get_connection().save_logbook(book)
- with jobboard(args.board_name, conf, persistence=backend) as jb:
- jb.post(args.job_name, book, details=store)
-
-
-def list_jobs(args):
- with jobboard(args.board_name, conf) as jb:
- for job in jb.iterjobs(ensure_fresh=True):
- print("%s - %s" % (job.uuid, job.details))
-
-
-def delete_job(args):
- with jobboard(args.board_name, conf) as jb:
- for job in jb.iterjobs(ensure_fresh=True):
- if job.uuid == args.job_uuid:
- jb.claim(job, "test-client")
- jb.consume(job, "test-client")
-
-
-def main(argv):
- parser = argparse.ArgumentParser()
- subparsers = parser.add_subparsers(title='subcommands',
- description='valid subcommands',
- help='additional help')
-
- # Consume command
- parser_consume = subparsers.add_parser('consume')
- parser_consume.add_argument('board_name')
- parser_consume.add_argument('job_uuid')
- parser_consume.set_defaults(func=consume_job)
-
- # Clear command
- parser_consume = subparsers.add_parser('clear')
- parser_consume.add_argument('board_name')
- parser_consume.set_defaults(func=clear_jobs)
-
- # Create command
- parser_create = subparsers.add_parser('create')
- parser_create.add_argument('board_name')
- parser_create.add_argument('job_name')
- parser_create.add_argument('details')
- parser_create.set_defaults(func=create_job)
-
- # Delete command
- parser_delete = subparsers.add_parser('delete')
- parser_delete.add_argument('board_name')
- parser_delete.add_argument('job_uuid')
- parser_delete.set_defaults(func=delete_job)
-
- # List command
- parser_list = subparsers.add_parser('list')
- parser_list.add_argument('board_name')
- parser_list.set_defaults(func=list_jobs)
-
- args = parser.parse_args(argv)
- args.func(args)
-
-if __name__ == '__main__':
- main(sys.argv[1:])
diff --git a/taskflow/examples/simple_linear_listening.py b/taskflow/examples/simple_linear_listening.py
index deff63c..850421b 100644
--- a/taskflow/examples/simple_linear_listening.py
+++ b/taskflow/examples/simple_linear_listening.py
@@ -92,11 +92,11 @@ engine = taskflow.engines.load(flow, store={
})
# This is where we attach our callback functions to the 2 different
-# notification objects that an engine exposes. The usage of a '*' (kleene star)
+# notification objects that an engine exposes. The usage of a ANY (kleene star)
# here means that we want to be notified on all state changes, if you want to
# restrict to a specific state change, just register that instead.
engine.notifier.register(ANY, flow_watch)
-engine.task_notifier.register(ANY, task_watch)
+engine.atom_notifier.register(ANY, task_watch)
# And now run!
engine.run()
diff --git a/taskflow/jobs/backends/impl_zookeeper.py b/taskflow/jobs/backends/impl_zookeeper.py
index 246416f..d92c2ba 100644
--- a/taskflow/jobs/backends/impl_zookeeper.py
+++ b/taskflow/jobs/backends/impl_zookeeper.py
@@ -21,6 +21,7 @@ import sys
import threading
from concurrent import futures
+import fasteners
from kazoo import exceptions as k_exceptions
from kazoo.protocol import paths as k_paths
from kazoo.recipe import watchers
@@ -35,7 +36,6 @@ from taskflow import logging
from taskflow import states
from taskflow.types import timing as tt
from taskflow.utils import kazoo_utils
-from taskflow.utils import lock_utils
from taskflow.utils import misc
LOG = logging.getLogger(__name__)
@@ -762,7 +762,7 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
def connected(self):
return self._connected and self._client.connected
- @lock_utils.locked(lock='_open_close_lock')
+ @fasteners.locked(lock='_open_close_lock')
def close(self):
if self._owned:
LOG.debug("Stopping client")
@@ -776,7 +776,7 @@ class ZookeeperJobBoard(base.NotifyingJobBoard):
LOG.debug("Stopped & cleared local state")
self._connected = False
- @lock_utils.locked(lock='_open_close_lock')
+ @fasteners.locked(lock='_open_close_lock')
def connect(self, timeout=10.0):
def try_clean():
diff --git a/taskflow/persistence/backends/impl_dir.py b/taskflow/persistence/backends/impl_dir.py
index b6d1a27..1047d67 100644
--- a/taskflow/persistence/backends/impl_dir.py
+++ b/taskflow/persistence/backends/impl_dir.py
@@ -60,6 +60,12 @@ class DirBackend(path_based.PathBasedBackend):
}
"""
+ DEFAULT_FILE_ENCODING = 'utf-8'
+ """
+ Default encoding used when decoding or encoding files into or from
+ text/unicode into binary or binary into text/unicode.
+ """
+
def __init__(self, conf):
super(DirBackend, self).__init__(conf)
max_cache_size = self._conf.get('max_cache_size')
@@ -71,7 +77,7 @@ class DirBackend(path_based.PathBasedBackend):
self.file_cache = cachetools.LRUCache(max_cache_size)
else:
self.file_cache = {}
- self.encoding = self._conf.get('encoding', 'utf-8')
+ self.encoding = self._conf.get('encoding', self.DEFAULT_FILE_ENCODING)
if not self._path:
raise ValueError("Empty path is disallowed")
self._path = os.path.abspath(self._path)
diff --git a/taskflow/persistence/backends/impl_memory.py b/taskflow/persistence/backends/impl_memory.py
index c18b471..37a67d0 100644
--- a/taskflow/persistence/backends/impl_memory.py
+++ b/taskflow/persistence/backends/impl_memory.py
@@ -274,14 +274,13 @@ class FakeFilesystem(object):
"""Link the destionation path to the source path."""
dest_path = self.normpath(dest_path)
src_path = self.normpath(src_path)
- dirname, basename = self.split(dest_path)
- parent_node = self._fetch_node(dirname, normalized=True)
- child_node = parent_node.find(basename,
- only_direct=True,
- include_self=False)
- if child_node is None:
- child_node = self._insert_child(parent_node, basename)
- child_node.metadata['target'] = src_path
+ try:
+ dest_node = self._fetch_node(dest_path, normalized=True)
+ except exc.NotFound:
+ parent_path, basename = self.split(dest_path)
+ parent_node = self._fetch_node(parent_path, normalized=True)
+ dest_node = self._insert_child(parent_node, basename)
+ dest_node.metadata['target'] = src_path
def __getitem__(self, path):
return self._get_item(self.normpath(path))
@@ -290,11 +289,11 @@ class FakeFilesystem(object):
path = self.normpath(path)
value = self._copier(value)
try:
- item_node = self._fetch_node(path, normalized=True)
- item_node.metadata.update(value=value)
+ node = self._fetch_node(path, normalized=True)
+ node.metadata.update(value=value)
except exc.NotFound:
- dirname, basename = self.split(path)
- parent_node = self._fetch_node(dirname, normalized=True)
+ parent_path, basename = self.split(path)
+ parent_node = self._fetch_node(parent_path, normalized=True)
self._insert_child(parent_node, basename, value=value)
diff --git a/taskflow/retry.py b/taskflow/retry.py
index b7135a9..3015c79 100644
--- a/taskflow/retry.py
+++ b/taskflow/retry.py
@@ -241,15 +241,20 @@ class Times(Retry):
"""Retries subflow given number of times. Returns attempt number."""
def __init__(self, attempts=1, name=None, provides=None, requires=None,
- auto_extract=True, rebind=None):
+ auto_extract=True, rebind=None, revert_all=False):
super(Times, self).__init__(name, provides, requires,
auto_extract, rebind)
self._attempts = attempts
+ if revert_all:
+ self._revert_action = REVERT_ALL
+ else:
+ self._revert_action = REVERT
+
def on_failure(self, history, *args, **kwargs):
if len(history) < self._attempts:
return RETRY
- return REVERT
+ return self._revert_action
def execute(self, history, *args, **kwargs):
return len(history) + 1
@@ -258,6 +263,16 @@ class Times(Retry):
class ForEachBase(Retry):
"""Base class for retries that iterate over a given collection."""
+ def __init__(self, name=None, provides=None, requires=None,
+ auto_extract=True, rebind=None, revert_all=False):
+ super(ForEachBase, self).__init__(name, provides, requires,
+ auto_extract, rebind)
+
+ if revert_all:
+ self._revert_action = REVERT_ALL
+ else:
+ self._revert_action = REVERT
+
def _get_next_value(self, values, history):
# Fetches the next resolution result to try, removes overlapping
# entries with what has already been tried and then returns the first
@@ -272,7 +287,7 @@ class ForEachBase(Retry):
try:
self._get_next_value(values, history)
except exc.NotFound:
- return REVERT
+ return self._revert_action
else:
return RETRY
@@ -285,9 +300,9 @@ class ForEach(ForEachBase):
"""
def __init__(self, values, name=None, provides=None, requires=None,
- auto_extract=True, rebind=None):
+ auto_extract=True, rebind=None, revert_all=False):
super(ForEach, self).__init__(name, provides, requires,
- auto_extract, rebind)
+ auto_extract, rebind, revert_all)
self._values = values
def on_failure(self, history, *args, **kwargs):
@@ -307,6 +322,12 @@ class ParameterizedForEach(ForEachBase):
each try.
"""
+ def __init__(self, name=None, provides=None, requires=None,
+ auto_extract=True, rebind=None, revert_all=False):
+ super(ParameterizedForEach, self).__init__(name, provides, requires,
+ auto_extract, rebind,
+ revert_all)
+
def on_failure(self, values, history, *args, **kwargs):
return self._on_failure(values, history)
diff --git a/taskflow/states.py b/taskflow/states.py
index c5ea579..265d6b2 100644
--- a/taskflow/states.py
+++ b/taskflow/states.py
@@ -69,10 +69,10 @@ _ALLOWED_JOB_TRANSITIONS = frozenset((
def check_job_transition(old_state, new_state):
- """Check that job can transition from old_state to new_state.
+ """Check that job can transition from from ``old_state`` to ``new_state``.
- If transition can be performed, it returns True. If transition
- should be ignored, it returns False. If transition is not
+ If transition can be performed, it returns true. If transition
+ should be ignored, it returns false. If transition is not
valid, it raises an InvalidState exception.
"""
if old_state == new_state:
@@ -138,10 +138,10 @@ _IGNORED_FLOW_TRANSITIONS = frozenset(
def check_flow_transition(old_state, new_state):
- """Check that flow can transition from old_state to new_state.
+ """Check that flow can transition from ``old_state`` to ``new_state``.
- If transition can be performed, it returns True. If transition
- should be ignored, it returns False. If transition is not
+ If transition can be performed, it returns true. If transition
+ should be ignored, it returns false. If transition is not
valid, it raises an InvalidState exception.
"""
if old_state == new_state:
@@ -171,18 +171,37 @@ _ALLOWED_TASK_TRANSITIONS = frozenset((
(REVERTING, FAILURE), # revert failed
(REVERTED, PENDING), # try again
-
- (SUCCESS, RETRYING), # retrying retry controller
- (RETRYING, RUNNING), # run retry controller that has been retrying
))
def check_task_transition(old_state, new_state):
- """Check that task can transition from old_state to new_state.
+ """Check that task can transition from ``old_state`` to ``new_state``.
- If transition can be performed, it returns True, False otherwise.
+ If transition can be performed, it returns true, false otherwise.
"""
pair = (old_state, new_state)
if pair in _ALLOWED_TASK_TRANSITIONS:
return True
return False
+
+
+# Retry state transitions
+# See: http://docs.openstack.org/developer/taskflow/states.html#retry
+
+_ALLOWED_RETRY_TRANSITIONS = list(_ALLOWED_TASK_TRANSITIONS)
+_ALLOWED_RETRY_TRANSITIONS.extend([
+ (SUCCESS, RETRYING), # retrying retry controller
+ (RETRYING, RUNNING), # run retry controller that has been retrying
+])
+_ALLOWED_RETRY_TRANSITIONS = frozenset(_ALLOWED_RETRY_TRANSITIONS)
+
+
+def check_retry_transition(old_state, new_state):
+ """Check that retry can transition from ``old_state`` to ``new_state``.
+
+ If transition can be performed, it returns true, false otherwise.
+ """
+ pair = (old_state, new_state)
+ if pair in _ALLOWED_RETRY_TRANSITIONS:
+ return True
+ return False
diff --git a/taskflow/tests/test_examples.py b/taskflow/tests/test_examples.py
index a7a297c..ce795dd 100644
--- a/taskflow/tests/test_examples.py
+++ b/taskflow/tests/test_examples.py
@@ -95,7 +95,7 @@ def iter_examples():
name, ext = os.path.splitext(filename)
if ext != ".py":
continue
- if not any(name.endswith(i) for i in ("utils", "no_test")):
+ if not name.endswith('utils'):
safe_name = safe_filename(name)
if safe_name:
yield name, safe_name
diff --git a/taskflow/tests/unit/action_engine/test_runner.py b/taskflow/tests/unit/action_engine/test_runner.py
index 98ae0e2..401cf50 100644
--- a/taskflow/tests/unit/action_engine/test_runner.py
+++ b/taskflow/tests/unit/action_engine/test_runner.py
@@ -45,8 +45,10 @@ class _RunnerTestMixin(object):
task_executor = executor.SerialTaskExecutor()
task_executor.start()
self.addCleanup(task_executor.stop)
- return runtime.Runtime(compilation, store,
- task_notifier, task_executor)
+ r = runtime.Runtime(compilation, store,
+ task_notifier, task_executor)
+ r.compile()
+ return r
class RunnerTest(test.TestCase, _RunnerTestMixin):
@@ -174,7 +176,7 @@ class RunnerTest(test.TestCase, _RunnerTestMixin):
rt.storage.get_atom_state(sad_tasks[0].name))
-class RunnerBuilderTest(test.TestCase, _RunnerTestMixin):
+class RunnerBuildTest(test.TestCase, _RunnerTestMixin):
def test_builder_manual_process(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(
@@ -182,8 +184,8 @@ class RunnerBuilderTest(test.TestCase, _RunnerTestMixin):
flow.add(*tasks)
rt = self._make_runtime(flow, initial_state=st.RUNNING)
- machine, memory = rt.runner.builder.build()
- self.assertTrue(rt.runner.builder.runnable())
+ machine, memory = rt.runner.build()
+ self.assertTrue(rt.runner.runnable())
self.assertRaises(fsm.NotInitialized, machine.process_event, 'poke')
# Should now be pending...
@@ -251,8 +253,8 @@ class RunnerBuilderTest(test.TestCase, _RunnerTestMixin):
flow.add(*tasks)
rt = self._make_runtime(flow, initial_state=st.RUNNING)
- machine, memory = rt.runner.builder.build()
- self.assertTrue(rt.runner.builder.runnable())
+ machine, memory = rt.runner.build()
+ self.assertTrue(rt.runner.runnable())
transitions = list(machine.run_iter('start'))
self.assertEqual((runner._UNDEFINED, st.RESUMING), transitions[0])
@@ -265,8 +267,8 @@ class RunnerBuilderTest(test.TestCase, _RunnerTestMixin):
flow.add(*tasks)
rt = self._make_runtime(flow, initial_state=st.RUNNING)
- machine, memory = rt.runner.builder.build()
- self.assertTrue(rt.runner.builder.runnable())
+ machine, memory = rt.runner.build()
+ self.assertTrue(rt.runner.runnable())
transitions = list(machine.run_iter('start'))
self.assertEqual((runner._GAME_OVER, st.FAILURE), transitions[-1])
@@ -278,8 +280,8 @@ class RunnerBuilderTest(test.TestCase, _RunnerTestMixin):
flow.add(*tasks)
rt = self._make_runtime(flow, initial_state=st.RUNNING)
- machine, memory = rt.runner.builder.build()
- self.assertTrue(rt.runner.builder.runnable())
+ machine, memory = rt.runner.build()
+ self.assertTrue(rt.runner.runnable())
transitions = list(machine.run_iter('start'))
self.assertEqual((runner._GAME_OVER, st.REVERTED), transitions[-1])
@@ -292,7 +294,7 @@ class RunnerBuilderTest(test.TestCase, _RunnerTestMixin):
flow.add(*tasks)
rt = self._make_runtime(flow, initial_state=st.RUNNING)
- machine, memory = rt.runner.builder.build()
+ machine, memory = rt.runner.build()
transitions = list(machine.run_iter('start'))
occurrences = dict((t, transitions.count(t)) for t in transitions)
diff --git a/taskflow/tests/unit/persistence/test_sql_persistence.py b/taskflow/tests/unit/persistence/test_sql_persistence.py
index 8489160..8a8e22c 100644
--- a/taskflow/tests/unit/persistence/test_sql_persistence.py
+++ b/taskflow/tests/unit/persistence/test_sql_persistence.py
@@ -40,18 +40,10 @@ PASSWD = "openstack_citest"
DATABASE = "tftest_" + ''.join(random.choice('0123456789')
for _ in range(12))
-try:
- from taskflow.persistence.backends import impl_sqlalchemy
-
- import sqlalchemy as sa
- SQLALCHEMY_AVAILABLE = True
-except Exception:
- SQLALCHEMY_AVAILABLE = False
-
-# Testing will try to run against these two mysql library variants.
-MYSQL_VARIANTS = ('mysqldb', 'pymysql')
+import sqlalchemy as sa
from taskflow.persistence import backends
+from taskflow.persistence.backends import impl_sqlalchemy
from taskflow import test
from taskflow.tests.unit.persistence import base
@@ -64,7 +56,7 @@ def _get_connect_string(backend, user, passwd, database=None, variant=None):
backend = "postgresql+%s" % (variant)
elif backend == "mysql":
if not variant:
- variant = 'mysqldb'
+ variant = 'pymysql'
backend = "mysql+%s" % (variant)
else:
raise Exception("Unrecognized backend: '%s'" % backend)
@@ -74,30 +66,24 @@ def _get_connect_string(backend, user, passwd, database=None, variant=None):
def _mysql_exists():
- if not SQLALCHEMY_AVAILABLE:
- return False
- for variant in MYSQL_VARIANTS:
- engine = None
- try:
- db_uri = _get_connect_string('mysql', USER, PASSWD,
- variant=variant)
- engine = sa.create_engine(db_uri)
- with contextlib.closing(engine.connect()):
- return True
- except Exception:
- pass
- finally:
- if engine is not None:
- try:
- engine.dispose()
- except Exception:
- pass
+ engine = None
+ try:
+ db_uri = _get_connect_string('mysql', USER, PASSWD)
+ engine = sa.create_engine(db_uri)
+ with contextlib.closing(engine.connect()):
+ return True
+ except Exception:
+ pass
+ finally:
+ if engine is not None:
+ try:
+ engine.dispose()
+ except Exception:
+ pass
return False
def _postgres_exists():
- if not SQLALCHEMY_AVAILABLE:
- return False
engine = None
try:
db_uri = _get_connect_string('postgres', USER, PASSWD, 'postgres')
@@ -114,7 +100,6 @@ def _postgres_exists():
pass
-@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available')
class SqlitePersistenceTest(test.TestCase, base.PersistenceTestMixin):
"""Inherits from the base test and sets up a sqlite temporary db."""
def _get_connection(self):
@@ -185,43 +170,26 @@ class BackendPersistenceTestMixin(base.PersistenceTestMixin):
" testing being skipped due to: %s" % (e))
-@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available')
@testtools.skipIf(not _mysql_exists(), 'mysql is not available')
class MysqlPersistenceTest(BackendPersistenceTestMixin, test.TestCase):
- def __init__(self, *args, **kwargs):
- test.TestCase.__init__(self, *args, **kwargs)
-
def _init_db(self):
- working_variant = None
- for variant in MYSQL_VARIANTS:
- engine = None
- try:
- db_uri = _get_connect_string('mysql', USER, PASSWD,
- variant=variant)
- engine = sa.create_engine(db_uri)
- with contextlib.closing(engine.connect()) as conn:
- conn.execute("CREATE DATABASE %s" % DATABASE)
- working_variant = variant
- except Exception:
- pass
- finally:
- if engine is not None:
- try:
- engine.dispose()
- except Exception:
- pass
- if working_variant:
- break
- if not working_variant:
- variants = ", ".join(MYSQL_VARIANTS)
- raise Exception("Failed to initialize MySQL db."
- " Tried these variants: %s; MySQL testing"
- " being skipped" % (variants))
- else:
- return _get_connect_string('mysql', USER, PASSWD,
- database=DATABASE,
- variant=working_variant)
+ engine = None
+ try:
+ db_uri = _get_connect_string('mysql', USER, PASSWD)
+ engine = sa.create_engine(db_uri)
+ with contextlib.closing(engine.connect()) as conn:
+ conn.execute("CREATE DATABASE %s" % DATABASE)
+ except Exception as e:
+ raise Exception('Failed to initialize MySQL db: %s' % (e))
+ finally:
+ if engine is not None:
+ try:
+ engine.dispose()
+ except Exception:
+ pass
+ return _get_connect_string('mysql', USER, PASSWD,
+ database=DATABASE)
def _remove_db(self):
engine = None
@@ -239,13 +207,9 @@ class MysqlPersistenceTest(BackendPersistenceTestMixin, test.TestCase):
pass
-@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available')
@testtools.skipIf(not _postgres_exists(), 'postgres is not available')
class PostgresPersistenceTest(BackendPersistenceTestMixin, test.TestCase):
- def __init__(self, *args, **kwargs):
- test.TestCase.__init__(self, *args, **kwargs)
-
def _init_db(self):
engine = None
try:
@@ -293,7 +257,6 @@ class PostgresPersistenceTest(BackendPersistenceTestMixin, test.TestCase):
pass
-@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available')
class SQLBackendFetchingTest(test.TestCase):
def test_sqlite_persistence_entry_point(self):
@@ -301,16 +264,16 @@ class SQLBackendFetchingTest(test.TestCase):
with contextlib.closing(backends.fetch(conf)) as be:
self.assertIsInstance(be, impl_sqlalchemy.SQLAlchemyBackend)
- @testtools.skipIf(not _postgres_exists(), 'postgres is not available')
+ @testtools.skipIf(not _mysql_exists(), 'mysql is not available')
def test_mysql_persistence_entry_point(self):
- uri = "mysql://%s:%s@localhost/%s" % (USER, PASSWD, DATABASE)
+ uri = _get_connect_string('mysql', USER, PASSWD, database=DATABASE)
conf = {'connection': uri}
with contextlib.closing(backends.fetch(conf)) as be:
self.assertIsInstance(be, impl_sqlalchemy.SQLAlchemyBackend)
- @testtools.skipIf(not _mysql_exists(), 'mysql is not available')
+ @testtools.skipIf(not _postgres_exists(), 'postgres is not available')
def test_postgres_persistence_entry_point(self):
- uri = "postgresql://%s:%s@localhost/%s" % (USER, PASSWD, DATABASE)
+ uri = _get_connect_string('postgres', USER, PASSWD, database=DATABASE)
conf = {'connection': uri}
with contextlib.closing(backends.fetch(conf)) as be:
self.assertIsInstance(be, impl_sqlalchemy.SQLAlchemyBackend)
diff --git a/taskflow/tests/unit/test_check_transition.py b/taskflow/tests/unit/test_check_transition.py
index bed7bc9..7c820fd 100644
--- a/taskflow/tests/unit/test_check_transition.py
+++ b/taskflow/tests/unit/test_check_transition.py
@@ -87,7 +87,7 @@ class CheckTaskTransitionTest(TransitionTest):
def test_from_success_state(self):
self.assertTransitions(from_state=states.SUCCESS,
- allowed=(states.REVERTING, states.RETRYING),
+ allowed=(states.REVERTING,),
ignored=(states.RUNNING, states.SUCCESS,
states.PENDING, states.FAILURE,
states.REVERTED))
@@ -112,6 +112,21 @@ class CheckTaskTransitionTest(TransitionTest):
states.RUNNING,
states.SUCCESS, states.FAILURE))
+
+class CheckRetryTransitionTest(CheckTaskTransitionTest):
+
+ def setUp(self):
+ super(CheckRetryTransitionTest, self).setUp()
+ self.check_transition = states.check_retry_transition
+ self.transition_exc_regexp = '^Retry transition.*not allowed'
+
+ def test_from_success_state(self):
+ self.assertTransitions(from_state=states.SUCCESS,
+ allowed=(states.REVERTING, states.RETRYING),
+ ignored=(states.RUNNING, states.SUCCESS,
+ states.PENDING, states.FAILURE,
+ states.REVERTED))
+
def test_from_retrying_state(self):
self.assertTransitions(from_state=states.RETRYING,
allowed=(states.RUNNING,),
diff --git a/taskflow/tests/unit/test_retries.py b/taskflow/tests/unit/test_retries.py
index edcc6d8..ddb256b 100644
--- a/taskflow/tests/unit/test_retries.py
+++ b/taskflow/tests/unit/test_retries.py
@@ -336,6 +336,66 @@ class RetryTest(utils.EngineTestBase):
'flow-1.f SUCCESS']
self.assertEqual(expected, capturer.values)
+ def test_nested_flow_with_retry_revert(self):
+ retry1 = retry.Times(0, 'r1', provides='x2')
+ flow = lf.Flow('flow-1').add(
+ utils.ProgressingTask("task1"),
+ lf.Flow('flow-2', retry1).add(
+ utils.ConditionalTask("task2", inject={'x': 1}))
+ )
+ engine = self._make_engine(flow)
+ engine.storage.inject({'y': 2})
+ with utils.CaptureListener(engine) as capturer:
+ try:
+ engine.run()
+ except Exception:
+ pass
+ self.assertEqual(engine.storage.fetch_all(), {'y': 2})
+ expected = ['flow-1.f RUNNING',
+ 'task1.t RUNNING',
+ 'task1.t SUCCESS(5)',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(1)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot!)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r REVERTING',
+ 'r1.r REVERTED',
+ 'flow-1.f REVERTED']
+ self.assertEqual(expected, capturer.values)
+
+ def test_nested_flow_with_retry_revert_all(self):
+ retry1 = retry.Times(0, 'r1', provides='x2', revert_all=True)
+ flow = lf.Flow('flow-1').add(
+ utils.ProgressingTask("task1"),
+ lf.Flow('flow-2', retry1).add(
+ utils.ConditionalTask("task2", inject={'x': 1}))
+ )
+ engine = self._make_engine(flow)
+ engine.storage.inject({'y': 2})
+ with utils.CaptureListener(engine) as capturer:
+ try:
+ engine.run()
+ except Exception:
+ pass
+ self.assertEqual(engine.storage.fetch_all(), {'y': 2})
+ expected = ['flow-1.f RUNNING',
+ 'task1.t RUNNING',
+ 'task1.t SUCCESS(5)',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(1)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot!)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r REVERTING',
+ 'r1.r REVERTED',
+ 'task1.t REVERTING',
+ 'task1.t REVERTED',
+ 'flow-1.f REVERTED']
+ self.assertEqual(expected, capturer.values)
+
def test_revert_all_retry(self):
flow = lf.Flow('flow-1', retry.Times(3, 'r1', provides='x')).add(
utils.ProgressingTask("task1"),
@@ -594,6 +654,108 @@ class RetryTest(utils.EngineTestBase):
'flow-1.f REVERTED']
self.assertItemsEqual(capturer.values, expected)
+ def test_nested_for_each_revert(self):
+ collection = [3, 2, 3, 5]
+ retry1 = retry.ForEach(collection, 'r1', provides='x')
+ flow = lf.Flow('flow-1').add(
+ utils.ProgressingTask("task1"),
+ lf.Flow('flow-2', retry1).add(
+ utils.FailingTaskWithOneArg('task2')
+ )
+ )
+ engine = self._make_engine(flow)
+ with utils.CaptureListener(engine) as capturer:
+ self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
+ expected = ['flow-1.f RUNNING',
+ 'task1.t RUNNING',
+ 'task1.t SUCCESS(5)',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(3)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot with 3)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(2)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot with 2)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(3)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot with 3)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(5)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot with 5)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r REVERTING',
+ 'r1.r REVERTED',
+ 'flow-1.f REVERTED']
+ self.assertEqual(expected, capturer.values)
+
+ def test_nested_for_each_revert_all(self):
+ collection = [3, 2, 3, 5]
+ retry1 = retry.ForEach(collection, 'r1', provides='x', revert_all=True)
+ flow = lf.Flow('flow-1').add(
+ utils.ProgressingTask("task1"),
+ lf.Flow('flow-2', retry1).add(
+ utils.FailingTaskWithOneArg('task2')
+ )
+ )
+ engine = self._make_engine(flow)
+ with utils.CaptureListener(engine) as capturer:
+ self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
+ expected = ['flow-1.f RUNNING',
+ 'task1.t RUNNING',
+ 'task1.t SUCCESS(5)',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(3)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot with 3)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(2)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot with 2)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(3)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot with 3)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(5)',
+ 'task2.t RUNNING',
+ 'task2.t FAILURE(Failure: RuntimeError: Woot with 5)',
+ 'task2.t REVERTING',
+ 'task2.t REVERTED',
+ 'r1.r REVERTING',
+ 'r1.r REVERTED',
+ 'task1.t REVERTING',
+ 'task1.t REVERTED',
+ 'flow-1.f REVERTED']
+ self.assertEqual(expected, capturer.values)
+
def test_for_each_empty_collection(self):
values = []
retry1 = retry.ForEach(values, 'r1', provides='x')
@@ -674,6 +836,95 @@ class RetryTest(utils.EngineTestBase):
'flow-1.f REVERTED']
self.assertItemsEqual(capturer.values, expected)
+ def test_nested_parameterized_for_each_revert(self):
+ values = [3, 2, 5]
+ retry1 = retry.ParameterizedForEach('r1', provides='x')
+ flow = lf.Flow('flow-1').add(
+ utils.ProgressingTask('task-1'),
+ lf.Flow('flow-2', retry1).add(
+ utils.FailingTaskWithOneArg('task-2')
+ )
+ )
+ engine = self._make_engine(flow)
+ engine.storage.inject({'values': values, 'y': 1})
+ with utils.CaptureListener(engine) as capturer:
+ self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
+ expected = ['flow-1.f RUNNING',
+ 'task-1.t RUNNING',
+ 'task-1.t SUCCESS(5)',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(3)',
+ 'task-2.t RUNNING',
+ 'task-2.t FAILURE(Failure: RuntimeError: Woot with 3)',
+ 'task-2.t REVERTING',
+ 'task-2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task-2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(2)',
+ 'task-2.t RUNNING',
+ 'task-2.t FAILURE(Failure: RuntimeError: Woot with 2)',
+ 'task-2.t REVERTING',
+ 'task-2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task-2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(5)',
+ 'task-2.t RUNNING',
+ 'task-2.t FAILURE(Failure: RuntimeError: Woot with 5)',
+ 'task-2.t REVERTING',
+ 'task-2.t REVERTED',
+ 'r1.r REVERTING',
+ 'r1.r REVERTED',
+ 'flow-1.f REVERTED']
+ self.assertEqual(expected, capturer.values)
+
+ def test_nested_parameterized_for_each_revert_all(self):
+ values = [3, 2, 5]
+ retry1 = retry.ParameterizedForEach('r1', provides='x',
+ revert_all=True)
+ flow = lf.Flow('flow-1').add(
+ utils.ProgressingTask('task-1'),
+ lf.Flow('flow-2', retry1).add(
+ utils.FailingTaskWithOneArg('task-2')
+ )
+ )
+ engine = self._make_engine(flow)
+ engine.storage.inject({'values': values, 'y': 1})
+ with utils.CaptureListener(engine) as capturer:
+ self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
+ expected = ['flow-1.f RUNNING',
+ 'task-1.t RUNNING',
+ 'task-1.t SUCCESS(5)',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(3)',
+ 'task-2.t RUNNING',
+ 'task-2.t FAILURE(Failure: RuntimeError: Woot with 3)',
+ 'task-2.t REVERTING',
+ 'task-2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task-2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(2)',
+ 'task-2.t RUNNING',
+ 'task-2.t FAILURE(Failure: RuntimeError: Woot with 2)',
+ 'task-2.t REVERTING',
+ 'task-2.t REVERTED',
+ 'r1.r RETRYING',
+ 'task-2.t PENDING',
+ 'r1.r RUNNING',
+ 'r1.r SUCCESS(5)',
+ 'task-2.t RUNNING',
+ 'task-2.t FAILURE(Failure: RuntimeError: Woot with 5)',
+ 'task-2.t REVERTING',
+ 'task-2.t REVERTED',
+ 'r1.r REVERTING',
+ 'r1.r REVERTED',
+ 'task-1.t REVERTING',
+ 'task-1.t REVERTED',
+ 'flow-1.f REVERTED']
+ self.assertEqual(expected, capturer.values)
+
def test_parameterized_for_each_empty_collection(self):
values = []
retry1 = retry.ParameterizedForEach('r1', provides='x')
diff --git a/taskflow/tests/unit/test_utils_lock_utils.py b/taskflow/tests/unit/test_utils_lock_utils.py
deleted file mode 100644
index 0c8213e..0000000
--- a/taskflow/tests/unit/test_utils_lock_utils.py
+++ /dev/null
@@ -1,281 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import collections
-import threading
-import time
-
-from taskflow import test
-from taskflow.test import mock
-from taskflow.utils import lock_utils
-from taskflow.utils import misc
-from taskflow.utils import threading_utils
-
-# NOTE(harlowja): Sleep a little so now() can not be the same (which will
-# cause false positives when our overlap detection code runs). If there are
-# real overlaps then they will still exist.
-NAPPY_TIME = 0.05
-
-# We will spend this amount of time doing some "fake" work.
-WORK_TIMES = [(0.01 + x / 100.0) for x in range(0, 5)]
-
-# Try to use a more accurate time for overlap detection (one that should
-# never go backwards and cause false positives during overlap detection...).
-now = misc.find_monotonic(allow_time_time=True)
-
-
-def _find_overlaps(times, start, end):
- overlaps = 0
- for (s, e) in times:
- if s >= start and e <= end:
- overlaps += 1
- return overlaps
-
-
-class MultilockTest(test.TestCase):
- THREAD_COUNT = 20
-
- def test_empty_error(self):
- self.assertRaises(ValueError,
- lock_utils.MultiLock, [])
- self.assertRaises(ValueError,
- lock_utils.MultiLock, ())
- self.assertRaises(ValueError,
- lock_utils.MultiLock, iter([]))
-
- def test_creation(self):
- locks = []
- for _i in range(0, 10):
- locks.append(threading.Lock())
- n_lock = lock_utils.MultiLock(locks)
- self.assertEqual(0, n_lock.obtained)
- self.assertEqual(len(locks), len(n_lock))
-
- def test_acquired(self):
- lock1 = threading.Lock()
- lock2 = threading.Lock()
- n_lock = lock_utils.MultiLock((lock1, lock2))
- self.assertTrue(n_lock.acquire())
- try:
- self.assertTrue(lock1.locked())
- self.assertTrue(lock2.locked())
- finally:
- n_lock.release()
- self.assertFalse(lock1.locked())
- self.assertFalse(lock2.locked())
-
- def test_acquired_context_manager(self):
- lock1 = threading.Lock()
- n_lock = lock_utils.MultiLock([lock1])
- with n_lock as gotten:
- self.assertTrue(gotten)
- self.assertTrue(lock1.locked())
- self.assertFalse(lock1.locked())
- self.assertEqual(0, n_lock.obtained)
-
- def test_partial_acquired(self):
- lock1 = threading.Lock()
- lock2 = mock.create_autospec(threading.Lock())
- lock2.acquire.return_value = False
- n_lock = lock_utils.MultiLock((lock1, lock2))
- with n_lock as gotten:
- self.assertFalse(gotten)
- self.assertTrue(lock1.locked())
- self.assertEqual(1, n_lock.obtained)
- self.assertEqual(2, len(n_lock))
- self.assertEqual(0, n_lock.obtained)
-
- def test_partial_acquired_failure(self):
- lock1 = threading.Lock()
- lock2 = mock.create_autospec(threading.Lock())
- lock2.acquire.side_effect = RuntimeError("Broke")
- n_lock = lock_utils.MultiLock((lock1, lock2))
- self.assertRaises(threading.ThreadError, n_lock.acquire)
- self.assertEqual(1, n_lock.obtained)
- n_lock.release()
-
- def test_release_failure(self):
- lock1 = threading.Lock()
- lock2 = mock.create_autospec(threading.Lock())
- lock2.acquire.return_value = True
- lock2.release.side_effect = RuntimeError("Broke")
- n_lock = lock_utils.MultiLock((lock1, lock2))
- self.assertTrue(n_lock.acquire())
- self.assertEqual(2, n_lock.obtained)
- self.assertRaises(threading.ThreadError, n_lock.release)
- self.assertEqual(2, n_lock.obtained)
- lock2.release.side_effect = None
- n_lock.release()
- self.assertEqual(0, n_lock.obtained)
-
- def test_release_partial_failure(self):
- lock1 = threading.Lock()
- lock2 = mock.create_autospec(threading.Lock())
- lock2.acquire.return_value = True
- lock2.release.side_effect = RuntimeError("Broke")
- lock3 = threading.Lock()
- n_lock = lock_utils.MultiLock((lock1, lock2, lock3))
- self.assertTrue(n_lock.acquire())
- self.assertEqual(3, n_lock.obtained)
- self.assertRaises(threading.ThreadError, n_lock.release)
- self.assertEqual(2, n_lock.obtained)
- lock2.release.side_effect = None
- n_lock.release()
- self.assertEqual(0, n_lock.obtained)
-
- def test_acquired_pass(self):
- activated = collections.deque()
- acquires = collections.deque()
- lock1 = threading.Lock()
- lock2 = threading.Lock()
- n_lock = lock_utils.MultiLock((lock1, lock2))
-
- def critical_section():
- start = now()
- time.sleep(NAPPY_TIME)
- end = now()
- activated.append((start, end))
-
- def run():
- with n_lock as gotten:
- acquires.append(gotten)
- critical_section()
-
- threads = []
- for _i in range(0, self.THREAD_COUNT):
- t = threading_utils.daemon_thread(run)
- threads.append(t)
- t.start()
- while threads:
- t = threads.pop()
- t.join()
-
- self.assertEqual(self.THREAD_COUNT, len(acquires))
- self.assertTrue(all(acquires))
- for (start, end) in activated:
- self.assertEqual(1, _find_overlaps(activated, start, end))
- self.assertFalse(lock1.locked())
- self.assertFalse(lock2.locked())
-
- def test_acquired_fail(self):
- activated = collections.deque()
- acquires = collections.deque()
- lock1 = threading.Lock()
- lock2 = threading.Lock()
- n_lock = lock_utils.MultiLock((lock1, lock2))
-
- def run():
- with n_lock as gotten:
- acquires.append(gotten)
- start = now()
- time.sleep(NAPPY_TIME)
- end = now()
- activated.append((start, end))
-
- def run_fail():
- try:
- with n_lock as gotten:
- acquires.append(gotten)
- raise RuntimeError()
- except RuntimeError:
- pass
-
- threads = []
- for i in range(0, self.THREAD_COUNT):
- if i % 2 == 1:
- target = run_fail
- else:
- target = run
- t = threading_utils.daemon_thread(target)
- threads.append(t)
- t.start()
- while threads:
- t = threads.pop()
- t.join()
-
- self.assertEqual(self.THREAD_COUNT, len(acquires))
- self.assertTrue(all(acquires))
- for (start, end) in activated:
- self.assertEqual(1, _find_overlaps(activated, start, end))
- self.assertFalse(lock1.locked())
- self.assertFalse(lock2.locked())
-
- def test_double_acquire_single(self):
- activated = collections.deque()
- acquires = []
-
- def run():
- start = now()
- time.sleep(NAPPY_TIME)
- end = now()
- activated.append((start, end))
-
- lock1 = threading.RLock()
- lock2 = threading.RLock()
- n_lock = lock_utils.MultiLock((lock1, lock2))
- with n_lock as gotten:
- acquires.append(gotten)
- run()
- with n_lock as gotten:
- acquires.append(gotten)
- run()
- run()
-
- self.assertTrue(all(acquires))
- self.assertEqual(2, len(acquires))
- for (start, end) in activated:
- self.assertEqual(1, _find_overlaps(activated, start, end))
-
- def test_double_acquire_many(self):
- activated = collections.deque()
- acquires = collections.deque()
- n_lock = lock_utils.MultiLock((threading.RLock(), threading.RLock()))
-
- def critical_section():
- start = now()
- time.sleep(NAPPY_TIME)
- end = now()
- activated.append((start, end))
-
- def run():
- with n_lock as gotten:
- acquires.append(gotten)
- critical_section()
- with n_lock as gotten:
- acquires.append(gotten)
- critical_section()
- critical_section()
-
- threads = []
- for i in range(0, self.THREAD_COUNT):
- t = threading_utils.daemon_thread(run)
- threads.append(t)
- t.start()
- while threads:
- t = threads.pop()
- t.join()
-
- self.assertTrue(all(acquires))
- self.assertEqual(self.THREAD_COUNT * 2, len(acquires))
- self.assertEqual(self.THREAD_COUNT * 3, len(activated))
- for (start, end) in activated:
- self.assertEqual(1, _find_overlaps(activated, start, end))
-
- def test_no_acquire_release(self):
- lock1 = threading.Lock()
- lock2 = threading.Lock()
- n_lock = lock_utils.MultiLock((lock1, lock2))
- self.assertRaises(threading.ThreadError, n_lock.release)
diff --git a/taskflow/utils/lock_utils.py b/taskflow/utils/lock_utils.py
deleted file mode 100644
index 7b1b026..0000000
--- a/taskflow/utils/lock_utils.py
+++ /dev/null
@@ -1,207 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This is a modified version of what was in oslo-incubator lockutils.py from
-# commit 5039a610355e5265fb9fbd1f4023e8160750f32e but this one does not depend
-# on oslo.cfg or the very large oslo-incubator oslo logging module (which also
-# pulls in oslo.cfg) and is reduced to only what taskflow currently wants to
-# use from that code.
-
-import contextlib
-import threading
-
-import six
-
-from taskflow import logging
-from taskflow.utils import misc
-
-LOG = logging.getLogger(__name__)
-
-
-@contextlib.contextmanager
-def try_lock(lock):
- """Attempts to acquire a lock, and auto releases if acquired (on exit)."""
- # NOTE(harlowja): the keyword argument for 'blocking' does not work
- # in py2.x and only is fixed in py3.x (this adjustment is documented
- # and/or debated in http://bugs.python.org/issue10789); so we'll just
- # stick to the format that works in both (oddly the keyword argument
- # works in py2.x but only with reentrant locks).
- was_locked = lock.acquire(False)
- try:
- yield was_locked
- finally:
- if was_locked:
- lock.release()
-
-
-def locked(*args, **kwargs):
- """A locking decorator.
-
- It will look for a provided attribute (typically a lock or a list
- of locks) on the first argument of the function decorated (typically this
- is the 'self' object) and before executing the decorated function it
- activates the given lock or list of locks as a context manager,
- automatically releasing that lock on exit.
-
- NOTE(harlowja): if no attribute name is provided then by default the
- attribute named '_lock' is looked for (this attribute is expected to be
- the lock/list of locks object/s) in the instance object this decorator
- is attached to.
- """
-
- def decorator(f):
- attr_name = kwargs.get('lock', '_lock')
-
- @six.wraps(f)
- def wrapper(self, *args, **kwargs):
- attr_value = getattr(self, attr_name)
- if isinstance(attr_value, (tuple, list)):
- lock = MultiLock(attr_value)
- else:
- lock = attr_value
- with lock:
- return f(self, *args, **kwargs)
-
- return wrapper
-
- # This is needed to handle when the decorator has args or the decorator
- # doesn't have args, python is rather weird here...
- if kwargs or not args:
- return decorator
- else:
- if len(args) == 1:
- return decorator(args[0])
- else:
- return decorator
-
-
-class MultiLock(object):
- """A class which attempts to obtain & release many locks at once.
-
- It is typically useful as a context manager around many locks (instead of
- having to nest individual lock context managers, which can become pretty
- awkward looking).
-
- NOTE(harlowja): The locks that will be obtained will be in the order the
- locks are given in the constructor, they will be acquired in order and
- released in reverse order (so ordering matters).
- """
-
- def __init__(self, locks):
- if not isinstance(locks, tuple):
- locks = tuple(locks)
- if len(locks) <= 0:
- raise ValueError("Zero locks requested")
- self._locks = locks
- self._local = threading.local()
-
- @property
- def _lock_stacks(self):
- # This is weird, but this is how thread locals work (in that each
- # thread will need to check if it has already created the attribute and
- # if not then create it and set it to the thread local variable...)
- #
- # This isn't done in the constructor since the constructor is only
- # activated by one of the many threads that could use this object,
- # and that means that the attribute will only exist for that one
- # thread.
- try:
- return self._local.stacks
- except AttributeError:
- self._local.stacks = []
- return self._local.stacks
-
- def __enter__(self):
- return self.acquire()
-
- @property
- def obtained(self):
- """Returns how many locks were last acquired/obtained."""
- try:
- return self._lock_stacks[-1]
- except IndexError:
- return 0
-
- def __len__(self):
- return len(self._locks)
-
- def acquire(self):
- """This will attempt to acquire all the locks given in the constructor.
-
- If all the locks can not be acquired (and say only X of Y locks could
- be acquired then this will return false to signify that not all the
- locks were able to be acquired, you can later use the :attr:`.obtained`
- property to determine how many were obtained during the last
- acquisition attempt).
-
- NOTE(harlowja): When not all locks were acquired it is still required
- to release since under partial acquisition the acquired locks
- must still be released. For example if 4 out of 5 locks were acquired
- this will return false, but the user **must** still release those
- other 4 to avoid causing locking issues...
- """
- gotten = 0
- for lock in self._locks:
- try:
- acked = lock.acquire()
- except (threading.ThreadError, RuntimeError) as e:
- # If we have already gotten some set of the desired locks
- # make sure we track that and ensure that we later release them
- # instead of losing them.
- if gotten:
- self._lock_stacks.append(gotten)
- raise threading.ThreadError(
- "Unable to acquire lock %s/%s due to '%s'"
- % (gotten + 1, len(self._locks), e))
- else:
- if not acked:
- break
- else:
- gotten += 1
- if gotten:
- self._lock_stacks.append(gotten)
- return gotten == len(self._locks)
-
- def __exit__(self, type, value, traceback):
- self.release()
-
- def release(self):
- """Releases any past acquired locks (partial or otherwise)."""
- height = len(self._lock_stacks)
- if not height:
- # Raise the same error type as the threading.Lock raises so that
- # it matches the behavior of the built-in class (it's odd though
- # that the threading.RLock raises a runtime error on this same
- # method instead...)
- raise threading.ThreadError('Release attempted on unlocked lock')
- # Cleans off one level of the stack (this is done so that if there
- # are multiple __enter__() and __exit__() pairs active that this will
- # only remove one level (the last one), and not all levels...
- for left in misc.countdown_iter(self._lock_stacks[-1]):
- lock_idx = left - 1
- lock = self._locks[lock_idx]
- try:
- lock.release()
- except (threading.ThreadError, RuntimeError) as e:
- # Ensure that we adjust the lock stack under failure so that
- # if release is attempted again that we do not try to release
- # the locks we already released...
- self._lock_stacks[-1] = left
- raise threading.ThreadError(
- "Unable to release lock %s/%s due to '%s'"
- % (left, len(self._locks), e))
- # At the end only clear it off, so that under partial failure we don't
- # lose any locks...
- self._lock_stacks.pop()
diff --git a/test-requirements.txt b/test-requirements.txt
index 16fc8b2..f25f6d4 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -16,7 +16,7 @@ zake>=0.1.6 # Apache-2.0
kazoo>=1.3.1,!=2.1
# Used for testing database persistence backends.
-SQLAlchemy>=0.9.7,<=0.9.99
+SQLAlchemy>=0.9.7,<1.1.0
alembic>=0.7.2
psycopg2
PyMySQL>=0.6.2 # MIT License
diff --git a/tools/state_graph.py b/tools/state_graph.py
index 3196140..997920d 100755
--- a/tools/state_graph.py
+++ b/tools/state_graph.py
@@ -49,12 +49,10 @@ def clean_event(name):
return name
-def make_machine(start_state, transitions, disallowed):
+def make_machine(start_state, transitions):
machine = fsm.FSM(start_state)
machine.add_state(start_state)
for (start_state, end_state) in transitions:
- if start_state in disallowed or end_state in disallowed:
- continue
if start_state not in machine:
machine.add_state(start_state)
if end_state not in machine:
@@ -125,30 +123,29 @@ def main():
if options.tasks:
source_type = "Tasks"
source = make_machine(states.PENDING,
- list(states._ALLOWED_TASK_TRANSITIONS),
- [states.RETRYING])
+ list(states._ALLOWED_TASK_TRANSITIONS))
elif options.retries:
source_type = "Retries"
source = make_machine(states.PENDING,
- list(states._ALLOWED_TASK_TRANSITIONS), [])
+ list(states._ALLOWED_RETRY_TRANSITIONS))
elif options.engines:
source_type = "Engines"
r = runner.Runner(DummyRuntime(), None)
- source, memory = r.builder.build()
+ source, memory = r.build()
internal_states.extend(runner._META_STATES)
ordering = 'out'
elif options.wbe_requests:
source_type = "WBE requests"
source = make_machine(protocol.WAITING,
- list(protocol._ALLOWED_TRANSITIONS), [])
+ list(protocol._ALLOWED_TRANSITIONS))
elif options.jobs:
source_type = "Jobs"
source = make_machine(states.UNCLAIMED,
- list(states._ALLOWED_JOB_TRANSITIONS), [])
+ list(states._ALLOWED_JOB_TRANSITIONS))
else:
source_type = "Flow"
source = make_machine(states.PENDING,
- list(states._ALLOWED_FLOW_TRANSITIONS), [])
+ list(states._ALLOWED_FLOW_TRANSITIONS))
graph_name = "%s states" % source_type
g = pydot.Dot(graph_name=graph_name, rankdir='LR',