selftest: Add basic sanity-check tests for nopython target

Previously we were only checking samba compiled OK with
--disable-python, not that it actually ran.

The main problem is all the make test framework is based around
subunit/smbtorture, neither of which we seem to build with
disable-python. However, for just a simple sanity-check, we can just
bypass all the subunit-filter work and just call the Perl code directly.
This works OK as long as it's just simple shell script tests that we're
running, as we can check the script's exit code directly.

The main thing that we really want to test is that we can start up the
smbd testenv and connect to it (i.e. a simple smbclient test).

This patch adds a new 'make test-nopython' target. This disables the
subunit filtering, and runs a small test-list that was generated manually.

Note that currently this has the limitation that it doesn't support known
failures or flapping tests. However, just checking that smd starts up OK
is probably OK for now.

Signed-off-by: Tim Beale <timbeale@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

Autobuild-User(master): Andrew Bartlett <abartlet@samba.org>
Autobuild-Date(master): Wed Feb 20 02:10:00 CET 2019 on sn-devel-144

4 files changed, 47 insertions, 1 deletions

diff --git a/Makefile b/Makefile
index c2de1df7b60..d456a410050 100644
--- a/Makefile
+++ b/Makefile
@@ -40,6 +40,9 @@ randomized-test:
 testlist:
 	$(WAF) test --list $(TEST_OPTIONS)
 
+test-nopython:
+	$(WAF) test --no-subunit-filter --test-list=selftest/no-python-tests.txt $(TEST_OPTIONS)
+
 dist:
 	touch .tmplock
 	WAFLOCK=.tmplock $(WAF) dist
diff --git a/script/autobuild.py b/script/autobuild.py
index f2a29497cf2..729f118019c 100755
--- a/script/autobuild.py
+++ b/script/autobuild.py
@@ -310,6 +310,7 @@ tasks = {
                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
                       ("make", "make -j", "text/plain"),
                       ("install", "make install", "text/plain"),
+                      ("test", "make test-nopython", "text/plain"),
                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
                       ("clean", "make clean", "text/plain"),
 
@@ -343,6 +344,7 @@ tasks = {
                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
                       ("make", "PYTHON=python2 make -j", "text/plain"),
                       ("install", "PYTHON=python2 make install", "text/plain"),
+                      ("test", "make test-nopython", "text/plain"),
                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 
diff --git a/selftest/no-python-tests.txt b/selftest/no-python-tests.txt
new file mode 100644
index 00000000000..33c2f0103ac
--- /dev/null
+++ b/selftest/no-python-tests.txt
@@ -0,0 +1,32 @@
+# A small subset of tests/testenvs to run as a sanity-check when samba
+# is built with --disable-python. One test-suite for each s3 testenv was
+# selected from 'python source3/selftest/tests.py' (and manually edited to
+# remove the filter-subunit portion of the command).
+# Notes:
+# - you cannot add knownfail tests to this list.
+# - only .sh tests supported here, and they must set and return $failed
+#   appropriately
+-- TEST --
+samba3.blackbox.smb2.not_casesensitive (fileserver)(fileserver)
+fileserver
+./source3/script/tests/test_smb2_not_casesensitive.sh //$SERVER/tmp $SERVER_IP $USERNAME $PASSWORD $LOCAL_PATH bin/smbclient || exit 1
+-- TEST --
+samba3.blackbox.rpcclient_srvsvc(simpleserver)
+simpleserver
+./source3/script/tests/test_rpcclientsrvsvc.sh $USERNAME $PASSWORD $SERVER bin/rpcclient tmp || exit 1
+-- TEST --
+samba3.blackbox.smbclient_large_file  krb5(ktest:local)
+ktest:local
+./source3/script/tests/test_smbclient_large_file.sh $PREFIX/ktest/krb5_ccache-3 bin/smbclient $SERVER $PREFIX  -k --configfile=$SMB_CONF_PATH || exit 1
+-- TEST --
+samba3.blackbox.smbclient_auth.plain (maptoguest) local creds(maptoguest)
+maptoguest
+./source3/script/tests/test_smbclient_auth.sh $SERVER $SERVER_IP $USERNAME $PASSWORD bin/smbclient --configfile=$SMB_CONF_PATH --option=clientntlmv2auth=no --option=clientlanmanauth=yes || exit 1
+-- TEST --
+samba3.blackbox.smbclient_auth.plain (nt4_dc) (nt4_dc)
+nt4_dc
+./source3/script/tests/test_smbclient_auth.sh $SERVER $SERVER_IP $DC_USERNAME $DC_PASSWORD bin/smbclient --configfile=$SMB_CONF_PATH  || exit 1
+-- TEST --
+samba3.blackbox.net_cred_change.(nt4_member:local)(nt4_member:local)
+nt4_member:local
+./source3/script/tests/test_net_cred_change.sh --configfile=$SMB_CONF_PATH || exit 1
diff --git a/selftest/wscript b/selftest/wscript
index b7eec2c2605..727adcf3126 100644
--- a/selftest/wscript
+++ b/selftest/wscript
@@ -90,6 +90,10 @@ def options(opt):
                   help=("use tests listed here, not defaults "
                         "(--test-list='FOO|' will execute FOO; "
                         "--test-list='FOO' will read it)"))
+    gr.add_option('--no-subunit-filter',
+                  help=("no (xfail) subunit filtering"),
+                  action="store_true", dest='NO_SUBUNIT_FILTER', default=False)
+
 
 def configure(conf):
     conf.env.SELFTEST_PREFIX = Options.options.SELFTEST_PREFIX
@@ -305,7 +309,12 @@ def cmd_testonly(opt):
     else:
         env.OPTIONS += ' --socket-wrapper'
         cmd = '(${CORE_COMMAND} && touch ${SELFTEST_PREFIX}/st_done) | ${FILTER_OPTIONS}'
-        if (os.environ.get('RUN_FROM_BUILD_FARM') is None and
+
+        if Options.options.NO_SUBUNIT_FILTER:
+            # Skip subunit filtering (i.e. because python is disabled).
+            # Use --one to bail out upon any failure
+            cmd = '(${CORE_COMMAND} --one && touch ${SELFTEST_PREFIX}/st_done)'
+        elif (os.environ.get('RUN_FROM_BUILD_FARM') is None and
             not Options.options.FILTERED_SUBUNIT):
             subunit_cache = os.path.join(env.SELFTEST_PREFIX, "subunit")
             cmd += ' | tee %s | ${FORMAT_TEST_OUTPUT}' % subunit_cache