Cleanup futility test coverage framework a bit

This cleans up the Makfile and test scripts a bit, and adds a new test for
the builtin commands.

BUG=chromium:224734
BRANCH=ToT
TEST=make runtests

Change-Id: Ibf5aa867d4dcabc0e46daac6633036b035c99ac8
Signed-off-by: Bill Richardson <wfrichar@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/207718

3 files changed, 60 insertions, 5 deletions

diff --git a/tests/futility/run_test_scripts.sh b/tests/futility/run_test_scripts.sh
index 97e9d841..38fc723b 100755
--- a/tests/futility/run_test_scripts.sh
+++ b/tests/futility/run_test_scripts.sh
@@ -7,7 +7,7 @@
 SCRIPTDIR=$(dirname $(readlink -f "$0"))
 . "$SCRIPTDIR/common.sh"
 
-# Mandatory arg is the path to the futility executable to test.
+# Mandatory arg is the directory where futility is installed.
 [ -z "${1:-}" ] && error "Directory argument is required"
 BINDIR="$1"
 shift
@@ -33,7 +33,10 @@ export SCRIPTDIR
 export OUTDIR
 
 # These are the scripts to run. Binaries are invoked directly by the Makefile.
-TESTS="${SCRIPTDIR}/test_dump_fmap.sh"
+TESTS="
+${SCRIPTDIR}/test_main.sh
+${SCRIPTDIR}/test_dump_fmap.sh
+"
 
 
 # Get ready...
@@ -45,14 +48,14 @@ progs=0
 # everything is built in (chromium:196079).
 
 # Here are the old programs to be wrapped
-# FIXME: dev_debug_vboot isn't tested right now.
+# TODO(crbug.com/224734): dev_debug_vboot isn't tested right now.
 PROGS=${*:-cgpt crossystem dev_sign_file dumpRSAPublicKey
            dump_kernel_config enable_dev_usb_boot gbb_utility
            tpm_init_temp_fix tpmc vbutil_firmware vbutil_kernel vbutil_key
            vbutil_keyblock vbutil_what_keys}
 
 # For now just compare results of invoking each program with no args.
-# FIXME(chromium-os:37062): Create true rigorous tests for every program.
+# TODO: Create true rigorous tests for every program.
 echo "-- old_bins --"
 for i in $PROGS; do
   : $(( progs++ ))
@@ -89,6 +92,12 @@ for i in $PROGS; do
   fi
 done
 
+# How many wrapped executables are left to incorporate? Did we check them all?
+xprogs=$(find ${OLDDIR} -type f -perm /111 | wc -l)
+if [ $xprogs -gt 0 ]; then
+  yellow "${progs}/${xprogs} wrapped executables tested"
+fi
+
 
 ##############################################################################
 # Invoke the scripts that test the builtin functions.
@@ -107,7 +116,7 @@ for i in $TESTS; do
   if [ ! "$rc" ]; then
     green "passed"
     : $(( pass++ ))
-    rm -f ${OUTDIR}/$i.{stdout,stderr,return}
+    rm -f ${OUTDIR}/$j.{stdout,stderr,return}
   else
     red "failed"
   fi
diff --git a/tests/futility/test_dump_fmap.sh b/tests/futility/test_dump_fmap.sh
index e4d35d08..1151fc46 100755
--- a/tests/futility/test_dump_fmap.sh
+++ b/tests/futility/test_dump_fmap.sh
@@ -27,4 +27,10 @@ cd "$OUTDIR"  # TODO: we really need a directory argument for dump_fmap.
 if "$FUTILITY" dump_fmap -x "${SCRIPTDIR}/data_fmap.bin" FMAP ; then
   echo Wait, that was supposed to fail. 1>&2
   exit 1
+else
+  rm -f FMAP
 fi
+
+# cleanup
+rm -f "$TMP"
+exit 0
diff --git a/tests/futility/test_main.sh b/tests/futility/test_main.sh
new file mode 100755
index 00000000..9bf7ca6d
--- /dev/null
+++ b/tests/futility/test_main.sh
@@ -0,0 +1,40 @@
+#!/bin/bash -eux
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+me=${0##*/}
+
+# Work in scratch directory
+cd "$OUTDIR"
+TMP="$me.tmp"
+
+# Built-in do-nothing commands.
+# TODO(crbug.com/224734): Remove these when we have enough built-in commands
+"$FUTILITY" foo hi
+"$FUTILITY" bar there
+"$FUTILITY" hey boys
+
+# No args returns nonzero exit code
+"$FUTILITY" && false
+
+"$FUTILITY" help > "$TMP"
+grep Usage "$TMP"
+# TODO(crbug.com/224734): Make sure all built-in commands have help, too.
+
+# It's weird but okay if the command is a full path.
+"$FUTILITY" /fake/path/to/help  > "$TMP"
+grep Usage "$TMP"
+
+# Make sure logging does something.
+# Note: This will zap any existing log file. Too bad.
+LOG="/tmp/futility.log"
+rm -f "$LOG"
+touch "$LOG"
+"$FUTILITY" help
+grep "$FUTILITY" "$LOG"
+rm "$LOG"
+
+# cleanup
+rm -f "$TMP"
+exit 0