Use the `state_graph.py` for all states diagrams

Switch to using the `state_graph.py` and dot output
as the source of all of the state diagrams (this
makes it easy for anyone to recreate them by just
running the script in the tools directory).

Also update the state diagram creator to have engine
states as well as retry states and replaces all
existing state diagrams with the updated prettified
versions.

Also adjusts some nits around wording and grammar that
were encountered during this updating process.

Change-Id: Ia783aed6c4136763e1e34cbd0b3e57ffb1109abe

2 files changed, 126 insertions, 43 deletions

diff --git a/tools/generate_states.sh b/tools/generate_states.sh
new file mode 100755
index 0000000..2da7581
--- /dev/null
+++ b/tools/generate_states.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -u
+xsltproc=`which xsltproc`
+if [ -z "$xsltproc" ]; then
+    echo "Please install xsltproc before continuing."
+    exit 1
+fi
+
+set -e
+if [ ! -d "$PWD/.diagram-tools" ]; then
+    git clone "https://github.com/vidarh/diagram-tools.git" "$PWD/.diagram-tools"
+fi
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+img_dir="$script_dir/../doc/source/img"
+
+echo "---- Updating task state diagram ----"
+python $script_dir/state_graph.py -t -f /tmp/states.svg
+$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/task_states.svg
+
+echo "---- Updating flow state diagram ----"
+python $script_dir/state_graph.py -f /tmp/states.svg
+$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/flow_states.svg
+
+echo "---- Updating engine state diagram ----"
+python $script_dir/state_graph.py -e -f /tmp/states.svg
+$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/engine_states.svg
+
+echo "---- Updating retry state diagram ----"
+python $script_dir/state_graph.py -r -f /tmp/states.svg
+$xsltproc $PWD/.diagram-tools/notugly.xsl /tmp/states.svg > $img_dir/retry_states.svg
diff --git a/tools/state_graph.py b/tools/state_graph.py
index f6a2057..77b8563 100644..100755
--- a/tools/state_graph.py
+++ b/tools/state_graph.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import optparse
 import os
 import sys
 
@@ -7,41 +8,13 @@ top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                        os.pardir))
 sys.path.insert(0, top_dir)
 
-import optparse
-import subprocess
-import tempfile
+import networkx as nx
+
+# To get this installed you may have to follow:
+# https://code.google.com/p/pydot/issues/detail?id=93 (until fixed).
+import pydot
 
 from taskflow import states
-from taskflow.types import graph as gr
-
-
-def mini_exec(cmd, ok_codes=(0,)):
-    stdout = subprocess.PIPE
-    stderr = subprocess.PIPE
-    proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, stdin=None)
-    (stdout, stderr) = proc.communicate()
-    rc = proc.returncode
-    if rc not in ok_codes:
-        raise RuntimeError("Could not run %s [%s]\nStderr: %s"
-                           % (cmd, rc, stderr))
-    return (stdout, stderr)
-
-
-def make_svg(graph, output_filename, output_format):
-    # NOTE(harlowja): requires pydot!
-    gdot = graph.export_to_dot()
-    if output_format == 'dot':
-        output = gdot
-    elif output_format in ('svg', 'svgz', 'png'):
-        with tempfile.NamedTemporaryFile(suffix=".dot") as fh:
-            fh.write(gdot)
-            fh.flush()
-            cmd = ['dot', '-T%s' % output_format, fh.name]
-            output, _stderr = mini_exec(cmd)
-    else:
-        raise ValueError('Unknown format: %s' % output_filename)
-    with open(output_filename, "wb") as fh:
-        fh.write(output)
 
 
 def main():
@@ -52,6 +25,14 @@ def main():
                       action='store_true',
                       help="use task state transitions",
                       default=False)
+    parser.add_option("-r", "--retries", dest="retries",
+                      action='store_true',
+                      help="use retry state transitions",
+                      default=False)
+    parser.add_option("-e", "--engines", dest="engines",
+                      action='store_true',
+                      help="use engine state transitions",
+                      default=False)
     parser.add_option("-T", "--format", dest="format",
                       help="output in given format",
                       default='svg')
@@ -60,20 +41,90 @@ def main():
     if options.filename is None:
         options.filename = 'states.%s' % options.format
 
-    g = gr.DiGraph(name="State transitions")
-    if not options.tasks:
-        source = states._ALLOWED_FLOW_TRANSITIONS
+    types = [options.engines, options.retries, options.tasks]
+    if sum([int(i) for i in types]) > 1:
+        parser.error("Only one of task/retry/engines may be specified.")
+
+    disallowed = set()
+    start_node = states.PENDING
+    if options.tasks:
+        source = list(states._ALLOWED_TASK_TRANSITIONS)
+        source_type = "Tasks"
+        disallowed.add(states.RETRYING)
+    elif options.retries:
+        source = list(states._ALLOWED_TASK_TRANSITIONS)
+        source_type = "Retries"
+    elif options.engines:
+        # TODO(harlowja): place this in states.py
+        source = [
+            (states.RESUMING, states.SCHEDULING),
+            (states.SCHEDULING, states.WAITING),
+            (states.WAITING, states.ANALYZING),
+            (states.ANALYZING, states.SCHEDULING),
+            (states.ANALYZING, states.WAITING),
+        ]
+        for u in (states.SCHEDULING, states.ANALYZING):
+            for v in (states.SUSPENDED, states.SUCCESS, states.REVERTED):
+                source.append((u, v))
+        source_type = "Engines"
+        start_node = states.RESUMING
     else:
-        source = states._ALLOWED_TASK_TRANSITIONS
+        source = list(states._ALLOWED_FLOW_TRANSITIONS)
+        source_type = "Flow"
+
+    transitions = nx.DiGraph()
     for (u, v) in source:
-        if not g.has_node(u):
-            g.add_node(u)
-        if not g.has_node(v):
-            g.add_node(v)
-        g.add_edge(u, v)
-    make_svg(g, options.filename, options.format)
+        if u not in disallowed:
+            transitions.add_node(u)
+        if v not in disallowed:
+            transitions.add_node(v)
+    for (u, v) in source:
+        if not transitions.has_node(u) or not transitions.has_node(v):
+            continue
+        transitions.add_edge(u, v)
+
+    graph_name = "%s states" % source_type
+    g = pydot.Dot(graph_name=graph_name, rankdir='LR',
+                  nodesep='0.25', overlap='false',
+                  ranksep="0.5", size="11x8.5",
+                  splines='true', ordering='in')
+    node_attrs = {
+        'fontsize': '11',
+    }
+    nodes = {}
+    nodes_order = []
+    edges_added = []
+    for (u, v) in nx.bfs_edges(transitions, source=start_node):
+        if u not in nodes:
+            nodes[u] = pydot.Node(u, **node_attrs)
+            g.add_node(nodes[u])
+            nodes_order.append(u)
+        if v not in nodes:
+            nodes[v] = pydot.Node(v, **node_attrs)
+            g.add_node(nodes[v])
+            nodes_order.append(v)
+    for u in nodes_order:
+        for v in transitions.successors_iter(u):
+            if (u, v) not in edges_added:
+                g.add_edge(pydot.Edge(nodes[u], nodes[v]))
+                edges_added.append((u, v))
+    start = pydot.Node("__start__", shape="point", width="0.1",
+                       xlabel='start', fontcolor='green', **node_attrs)
+    g.add_node(start)
+    g.add_edge(pydot.Edge(start, nodes[start_node], style='dotted'))
+
+    print("*" * len(graph_name))
+    print(graph_name)
+    print("*" * len(graph_name))
+    print(g.to_string().strip())
+
+    g.write(options.filename, format=options.format)
     print("Created %s at '%s'" % (options.format, options.filename))
 
+    # To make the svg more pretty use the following:
+    # $ xsltproc ../diagram-tools/notugly.xsl ./states.svg > pretty-states.svg
+    # Get diagram-tools from https://github.com/vidarh/diagram-tools.git
+
 
 if __name__ == '__main__':
     main()