Add framework for testing builtin futility functions.

This tweaks the Makefile and adds a couple of placeholder tests to prepare
for testing the builtin futility operations. There aren't any useful builtin
functions yet, but this lets us start adding them along with the tests.

BUG=chromium:224734
BRANCH=none
TEST=none

This doesn't actually do anything yet.

Change-Id: Iff0ca514f7d26346f072bd80a3bcd04621284843
Signed-off-by: Bill Richardson <wfrichar@chromium.org>
Reviewed-on: https://gerrit.chromium.org/gerrit/47432
Reviewed-by: Randall Spangler <rspangler@chromium.org>

4 files changed, 200 insertions, 0 deletions

diff --git a/tests/futility/common.sh b/tests/futility/common.sh
new file mode 100755
index 00000000..ed187d7b
--- /dev/null
+++ b/tests/futility/common.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Color output encodings.
+COL_RED='\E[31;1m'
+COL_GREEN='\E[32;1m'
+COL_YELLOW='\E[33;1m'
+COL_BLUE='\E[34;1m'
+COL_STOP='\E[0;m'
+
+# args: [message]
+green() {
+  echo -e "${COL_GREEN}$*${COL_STOP}"
+}
+
+# args: [message]
+yellow() {
+  echo -e "${COL_YELLOW}WARNING: $*${COL_STOP}"
+}
+
+# args: [message]
+red() {
+  echo -e "${COL_RED}$*${COL_STOP}"
+}
+
+# args: [nested level] [message]
+error() {
+  local lev=${1:-}
+  case "${1:-}" in
+    [0-9]*)
+      lev=$1
+      shift
+      ;;
+    *) lev=0
+      ;;
+  esac
+  local x=$(caller $lev)
+  local cline="${x%% *}"
+  local cfile="${x#* }"
+  cfile="${cfile##*/}"
+  local args="$*"
+  local spacer="${args:+: }"
+  red "at ${cfile}, line ${cline}${spacer}${args}" 1>&2
+  exit 1
+}
diff --git a/tests/futility/run_test_scripts.sh b/tests/futility/run_test_scripts.sh
new file mode 100755
index 00000000..d886ea95
--- /dev/null
+++ b/tests/futility/run_test_scripts.sh
@@ -0,0 +1,125 @@
+#!/bin/bash -eu
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Load common constants and variables.
+SCRIPTDIR=$(dirname $(readlink -f "$0"))
+. "$SCRIPTDIR/common.sh"
+
+# Mandatory arg is the path to the futility executable to test.
+[ -z "${1:-}" ] && error "Directory argument is required"
+BINDIR="$1"
+shift
+
+FUTILITY="$BINDIR/futility"
+OLDDIR="$BINDIR/old_bins"
+
+
+# The Makefile should export the $BUILD directory, but if it's not just warn
+# and guess (mostly so we can run the script manually).
+if [ -z "${BUILD:-}" ]; then
+  BUILD=$(dirname "${BINDIR}")
+  yellow "Assuming \$BUILD=$BUILD"
+fi
+OUTDIR="${BUILD}/tests/futility_test_results"
+[ -d "$OUTDIR" ] || mkdir -p "$OUTDIR"
+
+
+# Let each test know where to find things...
+export FUTILITY
+export SCRIPTDIR
+export OUTDIR
+
+# These are the scripts to run. Binaries are invoked directly by the Makefile.
+TESTS="${SCRIPTDIR}/test_not_really.sh"
+
+
+# Get ready...
+pass=0
+progs=0
+
+##############################################################################
+# But first, we'll just test the wrapped functions. This will go away when
+# everything is built in (chromium:196079).
+
+# Here are the old programs to be wrapped
+# FIXME: dev_debug_vboot isn't tested right now.
+PROGS=${*:-cgpt crossystem dev_sign_file dumpRSAPublicKey
+           dump_fmap dump_kernel_config enable_dev_usb_boot gbb_utility
+           tpm_init_temp_fix tpmc vbutil_firmware vbutil_kernel vbutil_key
+           vbutil_keyblock vbutil_what_keys}
+
+# For now just compare results of invoking each program with no args.
+# FIXME(chromium-os:37062): Create true rigorous tests for every program.
+echo "-- old_bins --"
+for i in $PROGS; do
+  : $(( progs++ ))
+
+  # Try the real thing first
+  echo -n "$i ... "
+  rc=$("${OLDDIR}/$i" \
+    1>"${OUTDIR}/$i.stdout.orig" 2>"${OUTDIR}/$i.stderr.orig" \
+    || echo "$?")
+  echo "${rc:-0}" > "${OUTDIR}/$i.return.orig"
+
+  # Then try the symlink
+  rc=$("$BINDIR/$i" 1>"${OUTDIR}/$i.stdout.link" \
+       2>"${OUTDIR}/$i.stderr.link" || echo "$?")
+  echo "${rc:-0}" > "${OUTDIR}/$i.return.link"
+
+  # And finally try the explicit wrapper
+  rc=$("$FUTILITY" "$i" 1>"${OUTDIR}/$i.stdout.futil" \
+       2>"${OUTDIR}/$i.stderr.futil" || echo "$?")
+  echo "${rc:-0}" > "${OUTDIR}/$i.return.futil"
+
+  # Different?
+  if cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.link" &&
+     cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.link" &&
+     cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.link" &&
+     cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.futil" &&
+     cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.futil" &&
+     cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.futil" ; then
+    green "passed"
+    : $(( pass++ ))
+    rm -f ${OUTDIR}/$i.{stdout,stderr,return}.{orig,link,futil}
+  else
+    red "failed"
+  fi
+done
+
+
+##############################################################################
+# Invoke the scripts that test the builtin functions.
+
+echo "-- builtin --"
+for i in $TESTS; do
+  j=${i##*/}
+
+  : $(( progs++ ))
+
+  echo -n "$j ... "
+  rm -f "${OUTDIR}/$j."*
+  rc=$("$i" "$FUTILITY" 1>"${OUTDIR}/$j.stdout" \
+       2>"${OUTDIR}/$j.stderr" || echo "$?")
+  echo "${rc:-0}" > "${OUTDIR}/$j.return"
+  if [ ! "$rc" ]; then
+    green "passed"
+    : $(( pass++ ))
+    rm -f ${OUTDIR}/$i.{stdout,stderr,return}
+  else
+    red "failed"
+  fi
+
+done
+
+##############################################################################
+# How'd we do?
+
+if [ "$pass" -eq "$progs" ]; then
+  green "Success: $pass / $progs passed"
+  exit 0
+fi
+
+red "FAIL: $pass / $progs passed"
+exit 1
diff --git a/tests/futility/test_not_really.c b/tests/futility/test_not_really.c
new file mode 100644
index 00000000..6a6555c4
--- /dev/null
+++ b/tests/futility/test_not_really.c
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+#include <stdio.h>
+#include "test_common.h"
+
+int main(int argc, char *argv[])
+{
+  TEST_EQ(0, 0, "Not Really A");
+
+  return !gTestSuccess;
+}
+
diff --git a/tests/futility/test_not_really.sh b/tests/futility/test_not_really.sh
new file mode 100755
index 00000000..8ea0c9a2
--- /dev/null
+++ b/tests/futility/test_not_really.sh
@@ -0,0 +1,13 @@
+#!/bin/bash -eu
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+me=${0##*/}
+
+TMP="$OUTDIR/$me.tmp"
+
+echo "FUTILITY=$FUTILITY" > "$TMP"
+echo "SCRIPTDIR=$SCRIPTDIR" >> "$TMP"
+
+exit 0