Cleanup futility test coverage framework a bit

This cleans up the Makfile and test scripts a bit, and adds a new test for
the builtin commands.

BUG=chromium:224734
BRANCH=ToT
TEST=make runtests

Change-Id: Ibf5aa867d4dcabc0e46daac6633036b035c99ac8
Signed-off-by: Bill Richardson <wfrichar@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/207718

5 files changed, 74 insertions, 11 deletions

diff --git a/Makefile b/Makefile
index 33ca966e..2f94d2e1 100644
--- a/Makefile
+++ b/Makefile
@@ -172,7 +172,7 @@ CFLAGS += -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64
 
 # Code coverage
 ifneq (${COV},)
-  COV_FLAGS = -O0 --coverage
+  COV_FLAGS = -O0 --coverage -DCOVERAGE
   CFLAGS += ${COV_FLAGS}
   LDFLAGS += ${COV_FLAGS}
   COV_INFO = ${BUILD}/coverage.info
@@ -758,8 +758,8 @@ ${BUILD}/firmware/linktest/main: ${FWLIB}
 ${BUILD}/firmware/linktest/main: LIBS = ${FWLIB}
 ALL_OBJS += ${BUILD}/firmware/linktest/main.o
 
-.phony: fwlinktest
-fwlinktest: ${FWLIB} \
+.PHONY: fwlinktest
+fwlinktest: \
 	${BUILD}/firmware/linktest/main_vbinit \
 	${BUILD}/firmware/linktest/main_vbsf \
 	${BUILD}/firmware/linktest/main
@@ -1227,9 +1227,9 @@ coverage_html:
 	lcov -a ${COV_INFO}.initial -a ${COV_INFO}.tests -o ${COV_INFO}.total
 	lcov -r ${COV_INFO}.total '/usr/*' '*/linktest/*' -o ${COV_INFO}.local
 	genhtml ${COV_INFO}.local -o ${BUILD}/coverage
-
-# Generate addtional coverage stats just for firmware subdir, because the
-# per-directory stats for the whole project don't include their own subdirs.
+# Generate addtional coverage stats just for firmware subdir, because the stats
+# for the whole project don't include subdirectory summaries. This will print
+# the summary for just the firmware sources.
 	lcov -r ${COV_INFO}.local '*/stub/*' -o ${COV_INFO}.nostub
 	lcov -e ${COV_INFO}.nostub '${SRCDIR}/firmware/*' \
 		-o ${COV_INFO}.firmware
diff --git a/futility/futility.c b/futility/futility.c
index 52c2d7dd..59145c3a 100644
--- a/futility/futility.c
+++ b/futility/futility.c
@@ -200,6 +200,10 @@ static void log_args(int argc, char *argv[])
 /******************************************************************************/
 /* Here we go */
 
+#ifdef COVERAGE
+void __gcov_flush(void);
+#endif
+
 int main(int argc, char *argv[], char *envp[])
 {
   char *progname;
@@ -278,6 +282,10 @@ int main(int argc, char *argv[], char *envp[])
   }
 
   fflush(0);
+#ifdef COVERAGE
+  /* Write gcov data prior to exec. */
+  __gcov_flush();
+#endif
   execve(oldname, argv, envp);
 
   fprintf(stderr, "%s failed to exec %s: %s\n", MYNAME,
diff --git a/tests/futility/run_test_scripts.sh b/tests/futility/run_test_scripts.sh
index 97e9d841..38fc723b 100755
--- a/tests/futility/run_test_scripts.sh
+++ b/tests/futility/run_test_scripts.sh
@@ -7,7 +7,7 @@
 SCRIPTDIR=$(dirname $(readlink -f "$0"))
 . "$SCRIPTDIR/common.sh"
 
-# Mandatory arg is the path to the futility executable to test.
+# Mandatory arg is the directory where futility is installed.
 [ -z "${1:-}" ] && error "Directory argument is required"
 BINDIR="$1"
 shift
@@ -33,7 +33,10 @@ export SCRIPTDIR
 export OUTDIR
 
 # These are the scripts to run. Binaries are invoked directly by the Makefile.
-TESTS="${SCRIPTDIR}/test_dump_fmap.sh"
+TESTS="
+${SCRIPTDIR}/test_main.sh
+${SCRIPTDIR}/test_dump_fmap.sh
+"
 
 
 # Get ready...
@@ -45,14 +48,14 @@ progs=0
 # everything is built in (chromium:196079).
 
 # Here are the old programs to be wrapped
-# FIXME: dev_debug_vboot isn't tested right now.
+# TODO(crbug.com/224734): dev_debug_vboot isn't tested right now.
 PROGS=${*:-cgpt crossystem dev_sign_file dumpRSAPublicKey
            dump_kernel_config enable_dev_usb_boot gbb_utility
            tpm_init_temp_fix tpmc vbutil_firmware vbutil_kernel vbutil_key
            vbutil_keyblock vbutil_what_keys}
 
 # For now just compare results of invoking each program with no args.
-# FIXME(chromium-os:37062): Create true rigorous tests for every program.
+# TODO: Create true rigorous tests for every program.
 echo "-- old_bins --"
 for i in $PROGS; do
   : $(( progs++ ))
@@ -89,6 +92,12 @@ for i in $PROGS; do
   fi
 done
 
+# How many wrapped executables are left to incorporate? Did we check them all?
+xprogs=$(find ${OLDDIR} -type f -perm /111 | wc -l)
+if [ $xprogs -gt 0 ]; then
+  yellow "${progs}/${xprogs} wrapped executables tested"
+fi
+
 
 ##############################################################################
 # Invoke the scripts that test the builtin functions.
@@ -107,7 +116,7 @@ for i in $TESTS; do
   if [ ! "$rc" ]; then
     green "passed"
     : $(( pass++ ))
-    rm -f ${OUTDIR}/$i.{stdout,stderr,return}
+    rm -f ${OUTDIR}/$j.{stdout,stderr,return}
   else
     red "failed"
   fi
diff --git a/tests/futility/test_dump_fmap.sh b/tests/futility/test_dump_fmap.sh
index e4d35d08..1151fc46 100755
--- a/tests/futility/test_dump_fmap.sh
+++ b/tests/futility/test_dump_fmap.sh
@@ -27,4 +27,10 @@ cd "$OUTDIR"  # TODO: we really need a directory argument for dump_fmap.
 if "$FUTILITY" dump_fmap -x "${SCRIPTDIR}/data_fmap.bin" FMAP ; then
   echo Wait, that was supposed to fail. 1>&2
   exit 1
+else
+  rm -f FMAP
 fi
+
+# cleanup
+rm -f "$TMP"
+exit 0
diff --git a/tests/futility/test_main.sh b/tests/futility/test_main.sh
new file mode 100755
index 00000000..9bf7ca6d
--- /dev/null
+++ b/tests/futility/test_main.sh
@@ -0,0 +1,40 @@
+#!/bin/bash -eux
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+me=${0##*/}
+
+# Work in scratch directory
+cd "$OUTDIR"
+TMP="$me.tmp"
+
+# Built-in do-nothing commands.
+# TODO(crbug.com/224734): Remove these when we have enough built-in commands
+"$FUTILITY" foo hi
+"$FUTILITY" bar there
+"$FUTILITY" hey boys
+
+# No args returns nonzero exit code
+"$FUTILITY" && false
+
+"$FUTILITY" help > "$TMP"
+grep Usage "$TMP"
+# TODO(crbug.com/224734): Make sure all built-in commands have help, too.
+
+# It's weird but okay if the command is a full path.
+"$FUTILITY" /fake/path/to/help  > "$TMP"
+grep Usage "$TMP"
+
+# Make sure logging does something.
+# Note: This will zap any existing log file. Too bad.
+LOG="/tmp/futility.log"
+rm -f "$LOG"
+touch "$LOG"
+"$FUTILITY" help
+grep "$FUTILITY" "$LOG"
+rm "$LOG"
+
+# cleanup
+rm -f "$TMP"
+exit 0