From 339f7e030cebe98a072d37acccbd1f0b4c1dea9b Mon Sep 17 00:00:00 2001 From: Bill Richardson Date: Fri, 5 Apr 2013 09:49:24 -0700 Subject: Add framework for testing builtin futility functions. This tweaks the Makefile and adds a couple of placeholder tests to prepare for testing the builtin futility operations. There aren't any useful builtin functions yet, but this lets us start adding them along with the tests. BUG=chromium:224734 BRANCH=none TEST=none This doesn't actually do anything yet. Change-Id: Iff0ca514f7d26346f072bd80a3bcd04621284843 Signed-off-by: Bill Richardson Reviewed-on: https://gerrit.chromium.org/gerrit/47432 Reviewed-by: Randall Spangler --- Makefile | 135 ++++++++++++++++++----------------- futility/tests/common.sh | 47 ------------ futility/tests/run_futility_tests.sh | 78 -------------------- tests/futility/common.sh | 47 ++++++++++++ tests/futility/run_test_scripts.sh | 125 ++++++++++++++++++++++++++++++++ tests/futility/test_not_really.c | 15 ++++ tests/futility/test_not_really.sh | 13 ++++ 7 files changed, 269 insertions(+), 191 deletions(-) delete mode 100755 futility/tests/common.sh delete mode 100755 futility/tests/run_futility_tests.sh create mode 100755 tests/futility/common.sh create mode 100755 tests/futility/run_test_scripts.sh create mode 100644 tests/futility/test_not_really.c create mode 100755 tests/futility/test_not_really.sh diff --git a/Makefile b/Makefile index 9ac96d3d..ec22984a 100644 --- a/Makefile +++ b/Makefile @@ -401,35 +401,35 @@ endif # These utilities should be linked statically. UTIL_NAMES_STATIC = \ - crossystem \ - dump_fmap \ - gbb_utility + utility/crossystem \ + utility/dump_fmap \ + utility/gbb_utility UTIL_NAMES = ${UTIL_NAMES_STATIC} \ - dev_sign_file \ - dump_kernel_config \ - dumpRSAPublicKey \ - tpm_init_temp_fix \ - tpmc \ - vbutil_firmware \ - vbutil_kernel \ - vbutil_key \ - vbutil_keyblock \ + utility/dev_sign_file \ + utility/dump_kernel_config \ + utility/dumpRSAPublicKey \ + utility/tpm_init_temp_fix \ + utility/tpmc \ + utility/vbutil_firmware \ + utility/vbutil_kernel \ + utility/vbutil_key \ + utility/vbutil_keyblock \ ifeq (${MINIMAL},) UTIL_NAMES += \ - bmpblk_font \ - bmpblk_utility \ - eficompress \ - efidecompress \ - load_kernel_test \ - pad_digest_utility \ - signature_digest_utility \ - verify_data + utility/bmpblk_font \ + utility/bmpblk_utility \ + utility/eficompress \ + utility/efidecompress \ + utility/load_kernel_test \ + utility/pad_digest_utility \ + utility/signature_digest_utility \ + utility/verify_data endif -UTIL_BINS_STATIC := $(addprefix ${BUILD}/utility/,${UTIL_NAMES_STATIC}) -UTIL_BINS = $(addprefix ${BUILD}/utility/,${UTIL_NAMES}) +UTIL_BINS_STATIC := $(addprefix ${BUILD}/,${UTIL_NAMES_STATIC}) +UTIL_BINS = $(addprefix ${BUILD}/,${UTIL_NAMES}) ALL_OBJS += $(addsuffix .o,${UTIL_BINS} ${UTIL_BINS_STATIC}) @@ -481,34 +481,35 @@ ALL_OBJS += ${TESTLIB_OBJS} # And some compiled tests. TEST_NAMES = \ - cgptlib_test \ - rollback_index2_tests \ - rollback_index3_tests \ - rsa_padding_test \ - rsa_utility_tests \ - rsa_verify_benchmark \ - sha_benchmark \ - sha_tests \ - stateful_util_tests \ - tlcl_tests \ - tpm_bootmode_tests \ - utility_string_tests \ - utility_tests \ - vboot_api_init_tests \ - vboot_api_devmode_tests \ - vboot_api_firmware_tests \ - vboot_api_kernel_tests \ - vboot_api_kernel2_tests \ - vboot_api_kernel3_tests \ - vboot_api_kernel4_tests \ - vboot_audio_tests \ - vboot_common_tests \ - vboot_common2_tests \ - vboot_common3_tests \ - vboot_display_tests \ - vboot_firmware_tests \ - vboot_kernel_tests \ - vboot_nvstorage_test + tests/cgptlib_test \ + tests/rollback_index2_tests \ + tests/rollback_index3_tests \ + tests/rsa_padding_test \ + tests/rsa_utility_tests \ + tests/rsa_verify_benchmark \ + tests/sha_benchmark \ + tests/sha_tests \ + tests/stateful_util_tests \ + tests/tlcl_tests \ + tests/tpm_bootmode_tests \ + tests/utility_string_tests \ + tests/utility_tests \ + tests/vboot_api_init_tests \ + tests/vboot_api_devmode_tests \ + tests/vboot_api_firmware_tests \ + tests/vboot_api_kernel_tests \ + tests/vboot_api_kernel2_tests \ + tests/vboot_api_kernel3_tests \ + tests/vboot_api_kernel4_tests \ + tests/vboot_audio_tests \ + tests/vboot_common_tests \ + tests/vboot_common2_tests \ + tests/vboot_common3_tests \ + tests/vboot_display_tests \ + tests/vboot_firmware_tests \ + tests/vboot_kernel_tests \ + tests/vboot_nvstorage_test \ + tests/futility/test_not_really # TODO: port these tests to new API, if not already eqivalent # functionality in other tests. These don't even compile at present. @@ -529,24 +530,23 @@ TEST_NAMES = \ # utility/load_firmware_test # And a few more... -TLCL_TESTS = \ - tpmtest_earlyextend \ - tpmtest_earlynvram \ - tpmtest_earlynvram2 \ - tpmtest_enable \ - tpmtest_fastenable \ - tpmtest_globallock \ - tpmtest_redefine_unowned \ - tpmtest_spaceperm \ - tpmtest_testsetup \ - tpmtest_timing \ - tpmtest_writelimit -TLCL_TEST_NAMES = $(addprefix tpm_lite/,${TLCL_TESTS}) -TLCL_TEST_BINS = $(addprefix ${BUILD}/tests/,${TLCL_TEST_NAMES}) +TLCL_TEST_NAMES = \ + tests/tpm_lite/tpmtest_earlyextend \ + tests/tpm_lite/tpmtest_earlynvram \ + tests/tpm_lite/tpmtest_earlynvram2 \ + tests/tpm_lite/tpmtest_enable \ + tests/tpm_lite/tpmtest_fastenable \ + tests/tpm_lite/tpmtest_globallock \ + tests/tpm_lite/tpmtest_redefine_unowned \ + tests/tpm_lite/tpmtest_spaceperm \ + tests/tpm_lite/tpmtest_testsetup \ + tests/tpm_lite/tpmtest_timing \ + tests/tpm_lite/tpmtest_writelimit TEST_NAMES += ${TLCL_TEST_NAMES} -TEST_BINS = $(addprefix ${BUILD}/tests/,${TEST_NAMES}) +# Finally +TEST_BINS = $(addprefix ${BUILD}/,${TEST_NAMES}) ALL_OBJS += $(addsuffix .o,${TEST_BINS}) # Directory containing test keys @@ -791,6 +791,7 @@ update_tlcl_structures: ${BUILD}/utility/tlcl_generator tests: ${TEST_BINS} ${TEST_BINS}: ${HOSTLIB} ${TESTLIB} +${TEST_BINS}: INCLUDES += -Itests ${TEST_BINS}: LIBS = ${HOSTLIB} ${TESTLIB} ${TESTLIB}: ${TESTLIB_OBJS} @@ -897,6 +898,7 @@ ALL_OBJS += ${BUILD}/firmware/lib/vboot_audio_for_test.o ${BUILD}/tests/rollback_index_test: INCLUDES += -I/usr/include ${BUILD}/tests/rollback_index_test: LIBS += -ltlcl +TLCL_TEST_BINS = $(addprefix ${BUILD}/,${TLCL_TEST_NAMES}) ${TLCL_TEST_BINS}: OBJS += ${BUILD}/tests/tpm_lite/tlcl_tests.o ${TLCL_TEST_BINS}: ${BUILD}/tests/tpm_lite/tlcl_tests.o ALL_OBJS += ${BUILD}/tests/tpm_lite/tlcl_tests.o @@ -995,7 +997,8 @@ runmisctests: test_setup .PHONY: runfutiltests runfutiltests: override DESTDIR = ${TEST_INSTALL_DIR} runfutiltests: test_setup install - futility/tests/run_futility_tests.sh ${DESTDIR} + tests/futility/run_test_scripts.sh ${DESTDIR} + ${RUNTEST} ${BUILD_RUN}/tests/futility/test_not_really # Run long tests, including all permutations of encryption keys (instead of # just the ones we use) and tests of currently-unused code. diff --git a/futility/tests/common.sh b/futility/tests/common.sh deleted file mode 100755 index ed187d7b..00000000 --- a/futility/tests/common.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Color output encodings. -COL_RED='\E[31;1m' -COL_GREEN='\E[32;1m' -COL_YELLOW='\E[33;1m' -COL_BLUE='\E[34;1m' -COL_STOP='\E[0;m' - -# args: [message] -green() { - echo -e "${COL_GREEN}$*${COL_STOP}" -} - -# args: [message] -yellow() { - echo -e "${COL_YELLOW}WARNING: $*${COL_STOP}" -} - -# args: [message] -red() { - echo -e "${COL_RED}$*${COL_STOP}" -} - -# args: [nested level] [message] -error() { - local lev=${1:-} - case "${1:-}" in - [0-9]*) - lev=$1 - shift - ;; - *) lev=0 - ;; - esac - local x=$(caller $lev) - local cline="${x%% *}" - local cfile="${x#* }" - cfile="${cfile##*/}" - local args="$*" - local spacer="${args:+: }" - red "at ${cfile}, line ${cline}${spacer}${args}" 1>&2 - exit 1 -} diff --git a/futility/tests/run_futility_tests.sh b/futility/tests/run_futility_tests.sh deleted file mode 100755 index ff2de4be..00000000 --- a/futility/tests/run_futility_tests.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Load common constants and variables. -. "$(dirname "$0")/common.sh" - -# Where are the programs I'm testing against? -[ -z "${1:-}" ] && error "Directory argument is required" -BINDIR="$1" -shift - -FUTILITY="$BINDIR/futility" -OLDDIR="$BINDIR/old_bins" - -BUILD=$(dirname "${BINDIR}") - -# Here are the old programs to be wrapped -# FIXME(chromium-os:37062): There are others besides these. -# FIXME: dev_debug_vboot isn't tested right now. -PROGS=${*:-cgpt crossystem dev_sign_file dumpRSAPublicKey - dump_fmap dump_kernel_config enable_dev_usb_boot gbb_utility - tpm_init_temp_fix tpmc vbutil_firmware vbutil_kernel vbutil_key - vbutil_keyblock vbutil_what_keys} - -# Get ready -pass=0 -progs=0 -pwd -OUTDIR="${BUILD}/tests/futility_test_dir" -[ -d "$OUTDIR" ] || mkdir -p "$OUTDIR" - -# For now just compare results of invoking each program with no args. -# FIXME(chromium-os:37062): Create true rigorous tests for every program. -for i in $PROGS; do - : $(( progs++ )) - - # Try the real thing first - echo -n "$i ... " - rc=$("${OLDDIR}/$i" \ - 1>"${OUTDIR}/$i.stdout.orig" 2>"${OUTDIR}/$i.stderr.orig" \ - || echo "$?") - echo "${rc:-0}" > "${OUTDIR}/$i.return.orig" - - # Then try the symlink - rc=$("$BINDIR/$i" 1>"${OUTDIR}/$i.stdout.link" \ - 2>"${OUTDIR}/$i.stderr.link" || echo "$?") - echo "${rc:-0}" > "${OUTDIR}/$i.return.link" - - # And finally try the explicit wrapper - rc=$("$FUTILITY" "$i" 1>"${OUTDIR}/$i.stdout.futil" \ - 2>"${OUTDIR}/$i.stderr.futil" || echo "$?") - echo "${rc:-0}" > "${OUTDIR}/$i.return.futil" - - # Different? - if cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.link" && - cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.link" && - cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.link" && - cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.futil" && - cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.futil" && - cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.futil" ; then - green "passed" - : $(( pass++ )) - rm -f ${OUTDIR}/$i.{stdout,stderr,return}.{orig,link,futil} - else - red "failed" - fi -done - -# done -if [ "$pass" -eq "$progs" ]; then - green "Success: $pass / $progs passed" - exit 0 -fi - -red "FAIL: $pass / $progs passed" -exit 1 diff --git a/tests/futility/common.sh b/tests/futility/common.sh new file mode 100755 index 00000000..ed187d7b --- /dev/null +++ b/tests/futility/common.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Color output encodings. +COL_RED='\E[31;1m' +COL_GREEN='\E[32;1m' +COL_YELLOW='\E[33;1m' +COL_BLUE='\E[34;1m' +COL_STOP='\E[0;m' + +# args: [message] +green() { + echo -e "${COL_GREEN}$*${COL_STOP}" +} + +# args: [message] +yellow() { + echo -e "${COL_YELLOW}WARNING: $*${COL_STOP}" +} + +# args: [message] +red() { + echo -e "${COL_RED}$*${COL_STOP}" +} + +# args: [nested level] [message] +error() { + local lev=${1:-} + case "${1:-}" in + [0-9]*) + lev=$1 + shift + ;; + *) lev=0 + ;; + esac + local x=$(caller $lev) + local cline="${x%% *}" + local cfile="${x#* }" + cfile="${cfile##*/}" + local args="$*" + local spacer="${args:+: }" + red "at ${cfile}, line ${cline}${spacer}${args}" 1>&2 + exit 1 +} diff --git a/tests/futility/run_test_scripts.sh b/tests/futility/run_test_scripts.sh new file mode 100755 index 00000000..d886ea95 --- /dev/null +++ b/tests/futility/run_test_scripts.sh @@ -0,0 +1,125 @@ +#!/bin/bash -eu +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Load common constants and variables. +SCRIPTDIR=$(dirname $(readlink -f "$0")) +. "$SCRIPTDIR/common.sh" + +# Mandatory arg is the path to the futility executable to test. +[ -z "${1:-}" ] && error "Directory argument is required" +BINDIR="$1" +shift + +FUTILITY="$BINDIR/futility" +OLDDIR="$BINDIR/old_bins" + + +# The Makefile should export the $BUILD directory, but if it's not just warn +# and guess (mostly so we can run the script manually). +if [ -z "${BUILD:-}" ]; then + BUILD=$(dirname "${BINDIR}") + yellow "Assuming \$BUILD=$BUILD" +fi +OUTDIR="${BUILD}/tests/futility_test_results" +[ -d "$OUTDIR" ] || mkdir -p "$OUTDIR" + + +# Let each test know where to find things... +export FUTILITY +export SCRIPTDIR +export OUTDIR + +# These are the scripts to run. Binaries are invoked directly by the Makefile. +TESTS="${SCRIPTDIR}/test_not_really.sh" + + +# Get ready... +pass=0 +progs=0 + +############################################################################## +# But first, we'll just test the wrapped functions. This will go away when +# everything is built in (chromium:196079). + +# Here are the old programs to be wrapped +# FIXME: dev_debug_vboot isn't tested right now. +PROGS=${*:-cgpt crossystem dev_sign_file dumpRSAPublicKey + dump_fmap dump_kernel_config enable_dev_usb_boot gbb_utility + tpm_init_temp_fix tpmc vbutil_firmware vbutil_kernel vbutil_key + vbutil_keyblock vbutil_what_keys} + +# For now just compare results of invoking each program with no args. +# FIXME(chromium-os:37062): Create true rigorous tests for every program. +echo "-- old_bins --" +for i in $PROGS; do + : $(( progs++ )) + + # Try the real thing first + echo -n "$i ... " + rc=$("${OLDDIR}/$i" \ + 1>"${OUTDIR}/$i.stdout.orig" 2>"${OUTDIR}/$i.stderr.orig" \ + || echo "$?") + echo "${rc:-0}" > "${OUTDIR}/$i.return.orig" + + # Then try the symlink + rc=$("$BINDIR/$i" 1>"${OUTDIR}/$i.stdout.link" \ + 2>"${OUTDIR}/$i.stderr.link" || echo "$?") + echo "${rc:-0}" > "${OUTDIR}/$i.return.link" + + # And finally try the explicit wrapper + rc=$("$FUTILITY" "$i" 1>"${OUTDIR}/$i.stdout.futil" \ + 2>"${OUTDIR}/$i.stderr.futil" || echo "$?") + echo "${rc:-0}" > "${OUTDIR}/$i.return.futil" + + # Different? + if cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.link" && + cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.link" && + cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.link" && + cmp -s "${OUTDIR}/$i.return.orig" "${OUTDIR}/$i.return.futil" && + cmp -s "${OUTDIR}/$i.stdout.orig" "${OUTDIR}/$i.stdout.futil" && + cmp -s "${OUTDIR}/$i.stderr.orig" "${OUTDIR}/$i.stderr.futil" ; then + green "passed" + : $(( pass++ )) + rm -f ${OUTDIR}/$i.{stdout,stderr,return}.{orig,link,futil} + else + red "failed" + fi +done + + +############################################################################## +# Invoke the scripts that test the builtin functions. + +echo "-- builtin --" +for i in $TESTS; do + j=${i##*/} + + : $(( progs++ )) + + echo -n "$j ... " + rm -f "${OUTDIR}/$j."* + rc=$("$i" "$FUTILITY" 1>"${OUTDIR}/$j.stdout" \ + 2>"${OUTDIR}/$j.stderr" || echo "$?") + echo "${rc:-0}" > "${OUTDIR}/$j.return" + if [ ! "$rc" ]; then + green "passed" + : $(( pass++ )) + rm -f ${OUTDIR}/$i.{stdout,stderr,return} + else + red "failed" + fi + +done + +############################################################################## +# How'd we do? + +if [ "$pass" -eq "$progs" ]; then + green "Success: $pass / $progs passed" + exit 0 +fi + +red "FAIL: $pass / $progs passed" +exit 1 diff --git a/tests/futility/test_not_really.c b/tests/futility/test_not_really.c new file mode 100644 index 00000000..6a6555c4 --- /dev/null +++ b/tests/futility/test_not_really.c @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2013 The Chromium OS Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ +#include +#include "test_common.h" + +int main(int argc, char *argv[]) +{ + TEST_EQ(0, 0, "Not Really A"); + + return !gTestSuccess; +} + diff --git a/tests/futility/test_not_really.sh b/tests/futility/test_not_really.sh new file mode 100755 index 00000000..8ea0c9a2 --- /dev/null +++ b/tests/futility/test_not_really.sh @@ -0,0 +1,13 @@ +#!/bin/bash -eu +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +me=${0##*/} + +TMP="$OUTDIR/$me.tmp" + +echo "FUTILITY=$FUTILITY" > "$TMP" +echo "SCRIPTDIR=$SCRIPTDIR" >> "$TMP" + +exit 0 -- cgit v1.2.1