tests: support hex encoded data and mqtt server

The mqtt server is started using a "random" port.

4 files changed, 132 insertions, 8 deletions

diff --git a/tests/FILEFORMAT b/tests/FILEFORMAT
index 653984905..44e79ab5f 100644
--- a/tests/FILEFORMAT
+++ b/tests/FILEFORMAT
@@ -37,7 +37,7 @@ which are treated together as a single identifier.
 </info>
 
 <reply>
-<data [nocheck="yes"] [sendzero="yes"] [base64="yes"]>
+<data [nocheck="yes"] [sendzero="yes"] [base64="yes"] [hex="yes"]>
 data to be sent to the client on its request and later verified that it arrived
 safely. Set nocheck="yes" to prevent the test script from verifying the arrival
 of this data.
@@ -60,6 +60,9 @@ of data encoded with base64. It is the only way a test case can contain binary
 data. (This attribute can in fact be used on any section, but it doesn't make
 much sense for other sections than "data").
 
+'hex' set to yes means that the data is a sequence of hex pairs. It will get
+decoded and used as "raw" data.
+
 For FTP file listings, the <data> section will be used *only* if you make sure
 that there has been a CWD done first to a directory named 'test-[num]' where
 [num] is the test case number. Otherwise the ftp server can't know from which
@@ -292,8 +295,8 @@ command is run. They are cleared again after the command has been run.
 Variables are first substituted as in the <command> section.
 </setenv>
 
-<command [option="no-output/no-include/force-output"] [timeout="secs"]
-         [delay="secs"][type="perl"]>
+<command [option="no-output/no-include/force-output/binary-trace"]
+         [timeout="secs"][delay="secs"][type="perl"]>
 command line to run, there's a bunch of %variables that get replaced
 accordingly.
 
@@ -322,6 +325,9 @@ otherwise written to verify stdout.
 Set option="no-include" to prevent the test script to slap on the --include
 argument.
 
+Set option="binary-trace" to use --trace instead of --trace-ascii for tracing.
+Suitable for binary-oriented protocols such as MQTT.
+
 Set timeout="secs" to override default server logs advisor read lock timeout.
 This timeout is used by the test harness, once that the command has completed
 execution, to wait for the test server to write out server side log files and
diff --git a/tests/getpart.pm b/tests/getpart.pm
index cd3b9e556..35ab30cb5 100644
--- a/tests/getpart.pm
+++ b/tests/getpart.pm
@@ -35,6 +35,15 @@ sub decode_base64 {
   return unpack("u", $len . $_);         # uudecode and print
 }
 
+sub decode_hex {
+    my $s = $_;
+    # remove everything not hex
+    $s =~ s/[^A-Fa-f0-9]//g;
+    # encode everything
+    $s =~ s/([a-fA-F0-9][a-fA-F0-9])/chr(hex($1))/eg;
+    return $s;
+}
+
 sub getpartattr {
     # if $part is undefined (ie only one argument) then
     # return the attributes of the section
@@ -81,6 +90,7 @@ sub getpart {
     my @this;
     my $inside=0;
     my $base64=0;
+    my $hex=0;
     my $line;
 
     for(@xml) {
@@ -96,6 +106,10 @@ sub getpart {
                 # attempt to detect our base64 encoded part
                 $base64=1;
             }
+            elsif($_ =~ /$part [^>]*hex=/) {
+                # attempt to detect a hex-encoded part
+                $hex=1;
+            }
             $inside++;
         }
         elsif(($inside >= 2) && ($_ =~ /^ *\<\/$part[ \>]/)) {
@@ -122,6 +136,13 @@ sub getpart {
                     $_ = $decoded;
                 }
             }
+            elsif($hex) {
+                # decode the whole array before returning it!
+                for(@this) {
+                    my $decoded = decode_hex($_);
+                    $_ = $decoded;
+                }
+            }
             return @this;
         }
         elsif($inside >= 2) {
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 6b1410e4d..c44b03fb5 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -78,6 +78,7 @@ use serverhelp qw(
     servername_str
     servername_canon
     server_pidfilename
+    server_portfilename
     server_logfilename
     );
 
@@ -123,6 +124,7 @@ my $base = 8990; # base port number
 my $minport;     # minimum used port number
 my $maxport;     # maximum used port number
 
+my $MQTTPORT;            # MQTT server port
 my $HTTPPORT;            # HTTP server port
 my $HTTP6PORT;           # HTTP IPv6 server port
 my $HTTPSPORT;           # HTTPS (stunnel) server port
@@ -326,6 +328,7 @@ my $run_event_based; # run curl with --test-event to test the event API
 my %run;          # running server
 my %doesntrun;    # servers that don't work, identified by pidfile
 my %serverpidfile;# all server pid file names, identified by server id
+my %serverportfile;# all server port file names, identified by server id
 my %runcert;      # cert file currently in use by an ssl running server
 
 # torture test variables
@@ -399,7 +402,8 @@ delete $ENV{'SSL_CERT_PATH'} if($ENV{'SSL_CERT_PATH'});
 delete $ENV{'CURL_CA_BUNDLE'} if($ENV{'CURL_CA_BUNDLE'});
 
 #######################################################################
-# Load serverpidfile hash with pidfile names for all possible servers.
+# Load serverpidfile and serverportfile hashes with file names for all
+# possible servers.
 #
 sub init_serverpidfile_hash {
   for my $proto (('ftp', 'http', 'imap', 'pop3', 'smtp', 'http/2')) {
@@ -409,17 +413,21 @@ sub init_serverpidfile_hash {
           my $serv = servername_id("$proto$ssl", $ipvnum, $idnum);
           my $pidf = server_pidfilename("$proto$ssl", $ipvnum, $idnum);
           $serverpidfile{$serv} = $pidf;
+          my $portf = server_portfilename("$proto$ssl", $ipvnum, $idnum);
+          $serverportfile{$serv} = $portf;
         }
       }
     }
   }
   for my $proto (('tftp', 'sftp', 'socks', 'ssh', 'rtsp', 'gopher', 'httptls',
-                  'dict', 'smb', 'smbs', 'telnet')) {
+                  'dict', 'smb', 'smbs', 'telnet', 'mqtt')) {
     for my $ipvnum ((4, 6)) {
       for my $idnum ((1, 2)) {
         my $serv = servername_id($proto, $ipvnum, $idnum);
         my $pidf = server_pidfilename($proto, $ipvnum, $idnum);
         $serverpidfile{$serv} = $pidf;
+        my $portf = server_portfilename($proto, $ipvnum, $idnum);
+        $serverportfile{$serv} = $portf;
       }
     }
   }
@@ -428,6 +436,8 @@ sub init_serverpidfile_hash {
       my $serv = servername_id("$proto$ssl", "unix", 1);
       my $pidf = server_pidfilename("$proto$ssl", "unix", 1);
       $serverpidfile{$serv} = $pidf;
+      my $portf = server_portfilename("$proto$ssl", "unix", 1);
+      $serverportfile{$serv} = $portf;
     }
   }
 }
@@ -2165,6 +2175,67 @@ sub runsshserver {
 #######################################################################
 # Start the socks server
 #
+sub runmqttserver {
+    my ($id, $verbose, $ipv6) = @_;
+    my $ip=$HOSTIP;
+    my $port = $MQTTPORT;
+    my $proto = 'mqtt';
+    my $ipvnum = 4;
+    my $idnum = ($id && ($id =~ /^(\d+)$/) && ($id > 1)) ? $id : 1;
+    my $server;
+    my $srvrname;
+    my $pidfile;
+    my $portfile;
+    my $logfile;
+    my $flags = "";
+
+    $server = servername_id($proto, $ipvnum, $idnum);
+    $pidfile = $serverpidfile{$server};
+    $portfile = $serverportfile{$server};
+
+    # don't retry if the server doesn't work
+    if ($doesntrun{$pidfile}) {
+        return (0,0);
+    }
+
+    my $pid = processexists($pidfile);
+    if($pid > 0) {
+        stopserver($server, "$pid");
+    }
+    unlink($pidfile) if(-f $pidfile);
+
+    $srvrname = servername_str($proto, $ipvnum, $idnum);
+
+    $logfile = server_logfilename($LOGDIR, $proto, $ipvnum, $idnum);
+
+    # start our MQTT server - on a random port!
+    my $cmd="server/mqttd".exe_ext('SRV').
+        " --port 0 ".
+        " --pidfile $pidfile".
+        " --portfile $portfile".
+        " --config $FTPDCMD";
+    my ($sockspid, $pid2) = startnew($cmd, $pidfile, 30, 0);
+
+    if($sockspid <= 0 || !pidexists($sockspid)) {
+        # it is NOT alive
+        logmsg "RUN: failed to start the $srvrname server\n";
+        stopserver($server, "$pid2");
+        $doesntrun{$pidfile} = 1;
+        return (0,0);
+    }
+
+    $MQTTPORT = pidfromfile($portfile);
+
+    if($verbose) {
+        logmsg "RUN: $srvrname server is now running PID $pid2 on PORT $MQTTPORT\n";
+    }
+
+    return ($pid2, $sockspid);
+}
+
+#######################################################################
+# Start the socks server
+#
 sub runsocksserver {
     my ($id, $verbose, $ipv6) = @_;
     my $ip=$HOSTIP;
@@ -3124,6 +3195,7 @@ sub subVariables {
   $$thing =~ s/%HTTP2PORT/$HTTP2PORT/g;
   $$thing =~ s/%HTTPPORT/$HTTPPORT/g;
   $$thing =~ s/%PROXYPORT/$HTTPPROXYPORT/g;
+  $$thing =~ s/%MQTTPORT/$MQTTPORT/g;
 
   $$thing =~ s/%IMAP6PORT/$IMAP6PORT/g;
   $$thing =~ s/%IMAPPORT/$IMAPPORT/g;
@@ -3684,9 +3756,14 @@ sub singletest {
         if((!$cmdhash{'option'}) || ($cmdhash{'option'} !~ /no-include/)) {
             $inc = " --include";
         }
-
         $cmdargs = "$out$inc ";
-        $cmdargs .= "--trace-ascii log/trace$testnum ";
+
+        if($cmdhash{'option'} && ($cmdhash{'option'} =~ /binary-trace/)) {
+            $cmdargs .= "--trace log/trace$testnum ";
+        }
+        else {
+            $cmdargs .= "--trace-ascii log/trace$testnum ";
+        }
         $cmdargs .= "--trace-time ";
         if($evbased) {
             $cmdargs .= "--test-event ";
@@ -4802,6 +4879,16 @@ sub startservers {
                 $run{'socks'}="$pid $pid2";
             }
         }
+        elsif($what eq "mqtt" ) {
+            if(!$run{'mqtt'}) {
+                ($pid, $pid2) = runmqttserver("", $verbose);
+                if($pid <= 0) {
+                    return "failed starting mqtt server";
+                }
+                printf ("* pid mqtt => %d %d\n", $pid, $pid2) if($verbose);
+                $run{'mqtt'}="$pid $pid2";
+            }
+        }
         elsif($what eq "http-unix") {
             if($torture && $run{'http-unix'} &&
                !responsive_http_server("http", $verbose, "unix", $HTTPUNIXPATH)) {
diff --git a/tests/serverhelp.pm b/tests/serverhelp.pm
index 4be207869..466091fcc 100644
--- a/tests/serverhelp.pm
+++ b/tests/serverhelp.pm
@@ -51,6 +51,7 @@ use vars qw(
     servername_str
     servername_canon
     server_pidfilename
+    server_portfilename
     server_logfilename
     server_cmdfilename
     server_inputfilename
@@ -105,7 +106,7 @@ sub servername_str {
 
     $proto = uc($proto) if($proto);
     die "unsupported protocol: '$proto'" unless($proto &&
-        ($proto =~ /^(((FTP|HTTP|HTTP\/2|IMAP|POP3|SMTP|HTTP-PIPE)S?)|(TFTP|SFTP|SOCKS|SSH|RTSP|GOPHER|HTTPTLS|DICT|SMB|SMBS|TELNET))$/));
+        ($proto =~ /^(((FTP|HTTP|HTTP\/2|IMAP|POP3|SMTP|HTTP-PIPE)S?)|(TFTP|SFTP|SOCKS|SSH|RTSP|GOPHER|HTTPTLS|DICT|SMB|SMBS|TELNET|MQTT))$/));
 
     $ipver = (not $ipver) ? 'ipv4' : lc($ipver);
     die "unsupported IP version: '$ipver'" unless($ipver &&
@@ -151,6 +152,15 @@ sub server_pidfilename {
     return '.'. servername_canon($proto, $ipver, $idnum) ."$trailer";
 }
 
+#***************************************************************************
+# Return file name for server port file.
+#
+sub server_portfilename {
+    my ($proto, $ipver, $idnum) = @_;
+    my $trailer = '_server.port';
+    return '.'. servername_canon($proto, $ipver, $idnum) ."$trailer";
+}
+
 
 #***************************************************************************
 # Return file name for server log file.