[v4_1_esv] ATF usage and documentation cleaned up

    Manual merge of rt38619 changes, minus autogenerated files

7 files changed, 176 insertions, 23 deletions

diff --git a/RELNOTES b/RELNOTES
index 2cc0c888..ee86f3a4 100644
--- a/RELNOTES
+++ b/RELNOTES
@@ -65,6 +65,21 @@ by Eric Young (eay@cryptsoft.com).
   the issue.
   [ISC-Bugs #37780]
 
+- Unit test execution now uses a path augmented during configuration
+  processing of the --with-atf option to locate ATF runtime tools, atf-run
+  and atf-report. For most installations of ATF, this should alleviate the
+  need to manually include them in the PATH, as was formerly required.
+  If the configure script cannot locate the tools it will emit a warning,
+  informing the user that the tools must be in the PATH when running unit
+  tests.
+  Secondly, please note that "make check" will now exit with a failure status
+  code (non-zero) if one or more unit tests fail.  This means that invoking
+  "make check" from an upper level directory will cause the make process to
+  STOP after the first test subdirectory with failed test(s).  To force all
+  tests in all subdirectories to run, regardless of individual test outcome,
+  use the command "make -k check".
+  [ISC-Bugs #38619]
+
 			Changes since 4.1.-ESV-R10
 
 - Corrected parser's right brace matching when a statement contains an error.
diff --git a/common/tests/Makefile.am b/common/tests/Makefile.am
index 2d72d51c..107a9295 100644
--- a/common/tests/Makefile.am
+++ b/common/tests/Makefile.am
@@ -21,7 +21,7 @@ ns_name_unittest_LDADD += ../libdhcp.a  \
 	../../omapip/libomapi.a ../../dst/libdst.a ../../minires/libres.a
 
 check: $(ATF_TESTS)
-	atf-run | atf-report
+	sh ${top_srcdir}/tests/unittest.sh
 
 endif
 
diff --git a/configure.ac b/configure.ac
index 4cd28dc1..76dd42a7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -216,24 +216,38 @@ if test "$atf_path" != "no" ; then
         do
             if test -f $d/lib/pkgconfig/atf-c.pc ; then
                 atf_pcp=$d/lib/pkgconfig
+                atf_path=$d
             elif test -f $d/lib64/pkgconfig/atf-c.pc ; then
                 atf_pcp=$d/lib64/pkgconfig
+                atf_path=$d
             fi
         done
     fi
+
     if test "$atf_pcp" = "" ; then
         AC_MSG_ERROR([Unable to find atf files in location specified])
     else
         ATF_CFLAGS="`PKG_CONFIG_PATH=$atf_pcp pkg-config --cflags atf-c` -DUNIT_TEST"
         ATF_LDFLAGS="`PKG_CONFIG_PATH=$atf_pcp pkg-config --libs atf-c`"
+        if  test -f $atf_pcp/atf-sh.pc ; then
+                ATF_BIN="`PKG_CONFIG_PATH=$atf_pcp pkg-config --variable=exec_prefix atf-sh`/bin"
+        else
+                # older versions don't have atf-sh, try usual place
+                ATF_BIN=$atf_path/bin
+        fi
+
+        if test ! -x $ATF_BIN/atf-run  -o  ! -x $ATF_BIN/atf-report ; then
+            AC_MSG_WARN([atf-run,atf-report not found, assuming they are in your path])
+        fi
+
         AC_SUBST(ATF_CFLAGS)
         AC_SUBST(ATF_LDFLAGS)
+        AC_SUBST(ATF_BIN)
     fi
 fi
 
 AM_CONDITIONAL(HAVE_ATF, test "$atf_pcp" != "")
-### Uncomment this once docs.lab.isc.org upgrades to automake 1.11
-### AM_COND_IF([HAVE_ATF], [AC_DEFINE([HAVE_ATF], [1], [ATF framework specified?])])
+AM_COND_IF([HAVE_ATF], [AC_DEFINE([HAVE_ATF], [1], [ATF framework specified?])])
 
 ###
 ### Path fun.  Older versions of DHCP were installed in /usr/sbin, so we
@@ -621,6 +635,7 @@ AC_OUTPUT([
   relay/Makefile
   server/Makefile
   tests/Makefile
+  tests/unittest.sh
   server/tests/Makefile
   doc/devel/doxyfile
 ])
@@ -655,6 +670,7 @@ if test "$atf_path" != "no"
 then
 echo "ATF_CFLAGS  : $ATF_CFLAGS" >> config.report
 echo "ATF_LDFLAGS : $ATF_LDFLAGS" >> config.report
+echo "ATF_BIN : $ATF_BIN" >> config.report
 echo
 fi
 
diff --git a/doc/devel/atf.dox b/doc/devel/atf.dox
index 1f3d7912..dd596841 100644
--- a/doc/devel/atf.dox
+++ b/doc/devel/atf.dox
@@ -30,18 +30,28 @@ Test Framework)</a> as a framework to run our unittests.
 
 ATF stands for Automated Test Framework, and is the framework used for unit
 tests in ISC DHCP and BIND9. ATF sources can be downloaded from
-http://code.google.com/p/kyua/wiki/ATF . ATF itself must be configured, compiled
+https://github.com/jmmv/kyua . ATF itself must be configured, compiled
 and then installed to be available during the DHCP configure procedure.  Please
 follow INSTALL file supplied with ATF sources (it's essentially the typical
 ./configure && make && make install procedure).
 
+Beginning with ATF version 0.16, it is necessary to include the following
+options --enable-tools and --disable-shared when configuring ATF:
+
+@verbatim
+    configure --prefix=<prefix> --enable-tools --disable-shared
+@endverbatim
+
+ISC DHCP unittests will run with ATF releases upto 0.19.  Beginning with
+ATF 0.20, the tools, atf-run and atf-report required by ISC DHCP, were
+deprecated and are no longer included with ATF.
+
 The ATF successor, called Kyua, is being developed. As of August 2012, the
 latest available release of Kyua is 0.5. It claims to offer feature parity with
 ATF. Migration to Kyua may be planned some time in the future, but DHCP uses ATF
-for now. Such an upgrade should be done in coordination with BIND. The latest
-tested version of ATF that DHCP's unittests were run against is 0.15.
+for now. Such an upgrade should be done in coordination with BIND.
 
-To build the unit-tests, use the following:
+To build and run the unit-tests, use the following:
 
 @verbatim
 $ ./configure --with-atf
@@ -49,6 +59,11 @@ $ make
 $ make check
 @endverbatim
 
+This will traverse the source tree running the unit tests in each unit test
+subdirectory.  Note that if one or more tests in a unit test subdirectory fail
+the make process will stop.  To run all of the tests regardless of outcome,
+use "make -k check"
+
 The following syntax is supported as well:
 @verbatim
 $ ./configure --with-atf=/path/to/your/atf/install
@@ -66,6 +81,9 @@ executable. The typical way to run tests is:
 
 @verbatim
 $ atf-run | atf-report
+(This assumes atf-run and atf-report are in your path)
+or
+$ sh ../../tests/unittests.sh
 @endverbatim
 
 atf-run will read the Atffile in the current directory and execute all the tests
diff --git a/server/tests/Makefile.am b/server/tests/Makefile.am
index ad23a751..b3fb5599 100644
--- a/server/tests/Makefile.am
+++ b/server/tests/Makefile.am
@@ -22,12 +22,8 @@ DHCPLIBS = $(top_builddir)/common/libdhcp.a $(top_builddir)/omapip/libomapi.a
           $(top_builddir)/minires/libres.a
 
 ATF_TESTS =
-TESTS = 
 if HAVE_ATF
 
-check: $(ATF_TESTS)
-	atf-run | atf-report
-
 ATF_TESTS += dhcpd_unittests legacy_unittests hash_unittests load_bal_unittests
 
 dhcpd_unittests_SOURCES = $(DHCPSRC)
@@ -49,6 +45,9 @@ legacy_unittests_LDADD = $(DHCPLIBS) $(ATF_LDFLAGS)
 load_bal_unittests_SOURCES = $(DHCPSRC) load_bal_unittest.c
 load_bal_unittests_LDADD = $(DHCPLIBS) $(ATF_LDFLAGS)
 
+check: $(ATF_TESTS)
+	sh ${top_srcdir}/tests/unittest.sh
+
 endif
 
-check_PROGRAMS = $(ATF_TESTS) $(TESTS)
+check_PROGRAMS = $(ATF_TESTS)
diff --git a/tests/HOWTO-unit-test b/tests/HOWTO-unit-test
index b84beba3..e00b7d58 100644
--- a/tests/HOWTO-unit-test
+++ b/tests/HOWTO-unit-test
@@ -13,7 +13,7 @@ and then opening doc/html/index.html
 Tests Overview
 --------------
 
-In DHCP, a unit test exercises a particular piece of code in 
+In DHCP, a unit test exercises a particular piece of code in
 isolation. There is a separate unit test per module or API. Each unit
 test lives in a directory beneath the code it is designed to exercise.
 So, we (will eventually) have:
@@ -26,31 +26,57 @@ So, we (will eventually) have:
 And so on.
 
 We are using ATF (Automated Test Framework) as a framework to run our
-unittests. See ISC DHCP Developer's Guide for much more thorough
+unit tests. See ISC DHCP Developer's Guide for much more thorough
 description of unit-test and ATF framework in general.
 
+Installing ATF
+--------------
+ATF sources can be downloaded from https://github.com/jmmv/kyua. ATF
+must be configured, compiled and then installed to be available during
+the DHCP configure procedure. Please follow INSTALL file supplied with
+ATF sources (it's essentially the typical ./configure && make &&
+make install procedure).
+
+Beginning with ATF version 0.16, it is necessary to include the following
+options --enable-tools and --disable-shared when configuring ATF:
+
+    configure --prefix=<prefix> --enable-tools --disable-shared
+
+ISC DHCP unittests will run with ATF releases upto 0.19.  Beginning with
+ATF 0.20, the tools, atf-run and atf-report required by ISC DHCP, were
+deprecated and are no longer included with ATF.
+
 Running Unit Tests
 ------------------
 
 In order to run the unit tests for DHCP, enable ATF support during configure:
 
-$ ./configure --with-atf
+$ ./configure --with-atf{=<atf-path>}
+
+    where <atf-path> is the path into which ATF was installed.  This would
+    be same value used for --prefix when ATF was configured (default is
+    /usr/local).
 
-And then use:
+And then build ISC_DHCP with:
+
+$ make
+
+Finally build and run unit tests with:
 
 $ make check
 
-This will run all of the unit tests. Make sure that ATF is actually
-installed and that you have atf-run and atf-report tool in your PATH.
+This will traverse the source tree running the unit tests in each unit test
+subdirectory.  Note that if one or more tests in a unit test subdirectory fail
+the make process will stop.  To run all of the tests regardless of outcome,
+use:
+
+$ make -k check
 
-You can run a single test by going to the appropriate test directory 
+You can run a single test by going to the appropriate test directory
 and invoking the test directly:
 
 $ cd server/tests
-$ atf-run | atf-report
-
-There are also a number of options that you can use when running a
-test. See atf-run and atf-report documentation.
+$ make check
 
 Adding a New Unit Test
 ----------------------
diff --git a/tests/unittest.sh.in b/tests/unittest.sh.in
new file mode 100755
index 00000000..0ee7ab70
--- /dev/null
+++ b/tests/unittest.sh.in
@@ -0,0 +1,79 @@
+#!/bin/sh
+#
+# Copyright (C) 2015  Internet Systems Consortium, Inc. ("ISC")
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+#
+# Script used to execute unit tests described by the Atffile in the current
+# directory.  It exits with return value of atf-run, which will be 0 if all
+# tests passed, non-zero otherwise.
+#
+
+# Add configured path to ATF tools, atf-run and atf-report
+PATH="@ATF_BIN@:${PATH}"
+export PATH
+
+# colors if not outputting to a dumb terminal and stdout is a tty
+if test "$TERM" != dumb && { test -t 1; } 2>/dev/null; then \
+    red='\033[0;31m'
+    green='\033[0;32m'
+    noclr='\033[0m'
+
+    # if echo supports -e, we must use it to set colors
+    # (output will be "" if its supported)
+    if [ -z "`echo -e`" ]
+    then
+        dash_e="-e"
+    fi
+fi;
+
+header="===================================================="
+
+status=0
+if [ -n "@ATF_BIN@" -a -f Atffile ]
+then
+    # run the tests
+    echo "Running unit tests..."
+	atf-run > atf.out
+	status=$?
+
+    # set color based on success/failure
+	if [ $status -eq 0 ]
+	then
+        color=$green
+	else
+        color=$red
+	fi
+
+    # spit out the test report
+    # We print everything upto the summary in
+    # "no color".  Print the summary in our
+    # result color.
+    cat atf.out | atf-report | while read line
+    do
+        cnt=`echo $line | grep -c "Summary"`
+        if [ $cnt -eq 1 ]
+        then
+            echo $dash_e $color$header
+        fi
+        echo $line;
+    done
+    echo $dash_e $header$noclr
+
+    # clean up unless there were test failures
+    if [ $status -eq 0 ]
+    then
+        rm -f atf.out
+    fi
+fi
+exit $status