autobahn: Various improvements:

- build: change autobahn into a auto-detected feature
- build: Remove duplication from existing test build file
- Avoid using __FILE__ and use GLibs test file utils
- Change indentation to be consistent with project
- Remove duplicate args to list number of tests or run specific tests
  glib provides this already
- In each test case run mainloop to completion to properly track case status
- Generate a report every run
- Output reports into the meson-logs directory, not the CWD of source
- Actually run the server until it is listening rather than sleep()

10 files changed, 297 insertions, 351 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 18de3d45..c0b9c547 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -42,7 +42,7 @@ fedora-autobahn:
       - "tests/autobahn"
   artifacts:
     paths:
-      - "_build/tests/autobahn/autobahn-report"
+      - "_build/meson-logs/autobahn-report"
 
 fedora-scan:
   extends: .build
diff --git a/meson.build b/meson.build
index 4c705a7a..8ab7c6df 100644
--- a/meson.build
+++ b/meson.build
@@ -281,7 +281,9 @@ if have_apache
   endif
 endif
 
-if not have_apache
+have_autobahn = find_program('wstest', required: get_option('autobahn')).found()
+
+if not have_apache or not have_autobahn
   warning('Some regression tests will not be compiled due to missing libraries or modules. Please check the logs for more details.')
 endif
 
@@ -411,7 +413,3 @@ summary({
   },
   section : 'Testing'
 )
-
-if get_option('autobahn')
-  subdir('tests/autobahn')
-endif
diff --git a/meson_options.txt b/meson_options.txt
index 6462fa9f..778e5d34 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -60,9 +60,9 @@ option('tests',
 )
 
 option('autobahn',
-  type: 'boolean',
-  value: 'true',
-  description: 'Enable autobahn client compilation'
+  type: 'feature',
+  value: 'auto',
+  description: 'Enable autobahn test suite'
 )
 
 option('installed_tests',
diff --git a/tests/autobahn/autobahn-server.json.in b/tests/autobahn/autobahn-server.json.in
new file mode 100644
index 00000000..02f86501
--- /dev/null
+++ b/tests/autobahn/autobahn-server.json.in
@@ -0,0 +1,8 @@
+
+{
+   "url": "ws://127.0.0.1:9001",
+   "outdir": "@MESON_BUILD_ROOT@/meson-logs/autobahn-reports",
+   "cases": ["*"],
+   "exclude-cases": [],
+   "exclude-agent-cases": {}
+}
diff --git a/tests/autobahn/autobahn-server.sh b/tests/autobahn/autobahn-server.sh
index 9596e72b..74046e75 100755
--- a/tests/autobahn/autobahn-server.sh
+++ b/tests/autobahn/autobahn-server.sh
@@ -2,12 +2,6 @@
 
 # set -x
 
-REPORTS_DIR="${PWD}/reports"
-
-cd "$(dirname "$0")"
-
-[ ! -d "${REPORTS_DIR}" ] && mkdir "${REPORTS_DIR}"
-
 ACTION=${1:---start}
 PORT=${2:-9001}
 
@@ -38,8 +32,7 @@ autobahn_start() {
              --name fuzzingserver \
              crossbario/autobahn-testsuite
    else
-      virtualenv ~/wstest
-      wstest -m fuzzingserver &> autobahn-server.log
+      wstest -m fuzzingserver -s 'autobahn-server.json'
    fi
 }
 
diff --git a/tests/autobahn/autobahn-test.c b/tests/autobahn/autobahn-test.c
new file mode 100644
index 00000000..406b76fc
--- /dev/null
+++ b/tests/autobahn/autobahn-test.c
@@ -0,0 +1,260 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/*
+ * soup-autobahn-test-client.c
+ *
+ * Copyright (C) 2021 Igalia S.L.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * SPDX-License-Identifier: LGPL-2.0-or-later
+ */
+
+#include "test-utils.h"
+
+#include <libsoup/soup.h>
+
+static char *address = "ws://localhost:9001";
+static char *agent = "libsoup";
+
+typedef void (*ConnectionFunc) (SoupWebsocketConnection *socket_connection,
+                                gint type,
+                                GBytes *message,
+                                gpointer data);
+
+typedef struct {
+        ConnectionFunc method;
+        gpointer data;
+        gboolean done;
+} ConnectionContext;
+
+typedef struct {
+        SoupSession *session;
+        unsigned int num_test_case;
+        char *path;
+} TestBundle;
+
+static void
+test_bundle_free (TestBundle *bundle)
+{
+        g_free (bundle->path);
+        g_free (bundle);
+}
+
+static void
+on_message_received (SoupWebsocketConnection *socket_connection,
+                     gint type, GBytes *message,
+                     gpointer data)
+{
+        ConnectionContext *ctx = (ConnectionContext *)data;
+
+        g_test_message ("Message recieved");
+
+        if (ctx && ctx->method)
+                ctx->method (socket_connection, type, message, ctx->data);
+}
+
+static void
+on_connection_closed (SoupWebsocketConnection *socket_connection,
+                      gpointer data)
+{
+        ConnectionContext *ctx = (ConnectionContext *)data;
+
+        g_test_message ("Connection closed");
+
+        g_object_unref (socket_connection);
+
+        ctx->done = TRUE;
+}
+
+static void
+on_connect (GObject *session,
+            GAsyncResult *res,
+            gpointer user_data)
+{
+        ConnectionContext *ctx = user_data;
+        GError *error = NULL;
+        SoupWebsocketConnection *socket_connection = soup_session_websocket_connect_finish (SOUP_SESSION (session), res, &error);
+        if (!socket_connection) {
+                g_test_message ("Connection failed: %s", error->message);
+                g_error_free (error);
+                ctx->done = TRUE;
+                return;
+        }
+
+        /* The performance tests increase the size of the payload up to 16 MB, let's disable
+       the limit to see what happens. */
+        soup_websocket_connection_set_max_incoming_payload_size (socket_connection, 0);
+
+        g_test_message ("Connected");
+        g_signal_connect (socket_connection, "message", G_CALLBACK (on_message_received), ctx);
+        g_signal_connect (socket_connection, "closed", G_CALLBACK (on_connection_closed), ctx);
+}
+
+static void
+connect_and_run (SoupSession *session, char *path, ConnectionFunc method, gpointer data)
+{
+        char *uri = g_strconcat (address, path, NULL);
+        SoupMessage *message = soup_message_new (SOUP_METHOD_GET, uri);
+        ConnectionContext *ctx = g_new0 (ConnectionContext, 1);
+        GMainContext *async_context = g_main_context_ref_thread_default ();
+
+        ctx->method = method;
+        ctx->data = data;
+        ctx->done = FALSE;
+
+        g_test_message ("Connecting to %s", uri);
+        soup_session_websocket_connect_async (session, message, NULL, NULL, G_PRIORITY_DEFAULT, NULL, on_connect, ctx);
+
+        while (!ctx->done)
+		g_main_context_iteration (async_context, TRUE);
+
+        g_object_unref (message);
+        g_free (uri);
+        g_free (ctx);
+        g_main_context_unref (async_context);
+}
+
+static void
+test_case_message_received (SoupWebsocketConnection *socket_connection,
+                            gint type,
+                            GBytes *message,
+                            gpointer data)
+{
+        /* Cannot send messages if we're not in an open state. */
+        if (soup_websocket_connection_get_state (socket_connection) != SOUP_WEBSOCKET_STATE_OPEN)
+                return;
+
+        g_test_message ("Sending message");
+
+        soup_websocket_connection_send_message (socket_connection, type, message);
+}
+
+static void
+test_case (gconstpointer data)
+{
+        TestBundle *bundle = (TestBundle *)data;
+
+        connect_and_run (bundle->session, bundle->path, test_case_message_received, bundle);
+}
+
+static void
+update_reports (SoupSession *session)
+{
+        char *path = g_strdup_printf ("/updateReports?agent=%s", agent);
+        g_test_message ("Updating reports...");
+        connect_and_run (session, path, NULL, NULL);
+        g_free (path);
+}
+
+static gboolean
+autobahn_server (const char *action, guint64 *num_cases_out)
+{
+        GSubprocessLauncher *launcher = g_subprocess_launcher_new (G_SUBPROCESS_FLAGS_STDOUT_PIPE);
+        GSubprocess *proc;
+        GError *error = NULL;
+        char *build_dir;
+        char *autobahn_script;
+
+        autobahn_script = g_test_build_filename (G_TEST_DIST, "autobahn", "autobahn-server.sh", NULL);
+        build_dir = g_test_build_filename (G_TEST_BUILT, "autobahn", NULL);
+
+        g_subprocess_launcher_set_cwd (launcher, build_dir);
+        proc = g_subprocess_launcher_spawn (launcher, &error, autobahn_script, action, NULL);
+
+        g_free (autobahn_script);
+        g_free (build_dir);
+        g_object_unref (launcher);
+
+        if (error) {
+                debug_printf (1, "Error running autobahn script: %s", error->message);
+                g_error_free (error);
+                return FALSE;
+        }
+
+        /* We are done if we are stopping the server */
+        if (strcmp (action, "--start"))
+                return TRUE;
+
+        GDataInputStream *stdout = g_data_input_stream_new  (g_subprocess_get_stdout_pipe (proc));
+        GRegex *re = g_regex_new ("Ok, will run (\\d+) test cases", 0, 0, NULL);
+        char *line = NULL;
+        gboolean ret = FALSE;
+
+        /* Read the process output until we know its listening successfully */
+        while (TRUE) {
+                line = g_data_input_stream_read_line_utf8 (stdout, NULL, NULL, NULL);
+                if (!line)
+                        goto done;
+
+                GMatchInfo *match;
+                if (g_regex_match (re, line, 0, &match)) {
+                        char *matched_number = g_match_info_fetch (match, 1);
+                        *num_cases_out = g_ascii_strtoull (matched_number, NULL, 10);
+
+                        ret = TRUE;
+                        g_free (matched_number);
+                        g_match_info_unref (match);
+                        goto done;
+                }
+
+                g_clear_pointer (&line, g_free);
+        }
+
+done:
+        g_free (line);
+        g_object_unref (stdout);
+        g_object_unref (proc);
+        g_regex_unref (re);
+
+        return ret;
+}
+
+int main (int argc, char *argv[])
+{
+        int ret = 0;
+        guint64 num_cases;
+        SoupSession *session;
+
+        test_init (argc, argv, NULL);
+
+        if (!autobahn_server ("--start", &num_cases))
+                exit (1);
+
+        session = soup_session_new ();
+        soup_session_add_feature_by_type (session, SOUP_TYPE_WEBSOCKET_EXTENSION_MANAGER);
+
+        for (int i = 0; i < num_cases; i++) {
+                char *test_path = g_strdup_printf ("/autobahn/%u", i);
+
+                TestBundle *bundle = g_new0 (TestBundle, 1);
+                bundle->session = session;
+                bundle->num_test_case = i;
+                bundle->path = g_strdup_printf ("/runCase?case=%u&agent=%s", i, agent);
+
+                g_test_add_data_func_full (test_path, bundle, test_case, (GDestroyNotify)test_bundle_free);
+
+                g_free (test_path);
+        }
+
+        ret = g_test_run ();
+
+        update_reports (session);
+
+        g_object_unref (session);
+
+        autobahn_server ("--stop", NULL);
+        test_cleanup ();
+
+        return ret;
+}
diff --git a/tests/autobahn/fuzzingserver.json b/tests/autobahn/fuzzingserver.json
deleted file mode 100644
index 4b7fe3bd..00000000
--- a/tests/autobahn/fuzzingserver.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "url": "ws://127.0.0.1:9001",
-    "outdir": "./reports/clients",
-    "cases": ["*"],
-    "exclude-cases": [],
-    "exclude-agent-cases": {}
-}
-
diff --git a/tests/autobahn/meson.build b/tests/autobahn/meson.build
index 9c158ad2..c4598bd4 100644
--- a/tests/autobahn/meson.build
+++ b/tests/autobahn/meson.build
@@ -1,26 +1,21 @@
-test_utils_name = 'test-utils'
-installed_tests_execdir = join_paths(get_option('libexecdir'), 'installed-tests', libsoup_api_name)
-installed_tests_enabled = get_option('installed_tests')
-abs_installed_tests_execdir = join_paths(prefix, installed_tests_execdir)
+autobahn_conf = configuration_data()
+autobahn_conf.set('MESON_BUILD_ROOT', meson.build_root())
 
-if cc.get_id() == 'msvc'
-  test_utils = static_library(test_utils_name, '../' + test_utils_name + '.c',
-    dependencies : libsoup_static_dep)
-else
-  test_utils = library(test_utils_name, '../' + test_utils_name + '.c',
-    dependencies : libsoup_static_dep,
-    install : installed_tests_enabled,
-    install_dir : installed_tests_execdir,
-  )
-endif
+configure_file(
+  input : 'autobahn-server.json.in',
+  output : 'autobahn-server.json',
+  configuration : autobahn_conf,
+)
 
-deps = [
-  glib_deps,
-  libsoup_dep
-]
+client = executable('autobahn-test', 'autobahn-test.c',
+    link_with : test_utils,
+    dependencies : libsoup_static_dep,
+    include_directories : include_directories('..'),
+)
 
-client = executable('soup-autobahn-test-client', 'soup-autobahn-test-client.c',
-    dependencies: deps,
-    link_with : test_utils
+test('autobahn-test', client,
+  suite : 'autobahn',
+  env : env,
+  timeout : 3600,
+  protocol : 'tap',
 )
-test('soup-autobahn-test-client', client, suite: 'autobahn', timeout: 3600)
diff --git a/tests/autobahn/soup-autobahn-test-client.c b/tests/autobahn/soup-autobahn-test-client.c
deleted file mode 100644
index 047149d4..00000000
--- a/tests/autobahn/soup-autobahn-test-client.c
+++ /dev/null
@@ -1,304 +0,0 @@
-/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
-/*
- * soup-autobahn-test-client.c
- *
- * Copyright (C) 2021 Igalia S.L.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Library General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Library General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * SPDX-License-Identifier: LGPL-2.0-or-later
- */
-
-#include "../test-utils.h"
-
-#include <stdio.h>
-#include <unistd.h>
-#include <libsoup/soup.h>
-
-GMainLoop *loop;
-static char *address = "ws://localhost:9001";
-static char *agent = "libsoup";
-static unsigned int total_num_cases = 0;
-
-typedef void (*ConnectionFunc) (SoupWebsocketConnection *socket_connection,
-                                gint type,
-                                GBytes *message,
-                                gpointer data);
-
-typedef struct {
-    ConnectionFunc method;
-    gpointer data;
-} ConnectionContext;
-
-static void run_case (SoupSession *session, const unsigned int test_case);
-
-typedef struct {
-    SoupSession *session;
-    unsigned int num_test_case;
-    char *path;
-} TestBundle;
-
-static gboolean option_run_all = FALSE;
-static int option_run_test = -1;
-static gboolean option_number_of_tests = FALSE;
-static gboolean option_update_report = FALSE;
-
-static GOptionEntry entries[] =
-{
-    { "run-all",         'a', 0, G_OPTION_ARG_NONE, &option_run_all,         "Run all tests", NULL },
-    { "test",            't', 0, G_OPTION_ARG_INT,  &option_run_test,        "Run TEST only", "TEST" },
-    { "number-of-tests", 'n', 0, G_OPTION_ARG_NONE, &option_number_of_tests, "Queries the Autobahn server for the number of test cases", NULL },
-    { "update-report",   'r', 0, G_OPTION_ARG_NONE, &option_update_report,   "Requests the Autobahn server to update the report for tests", NULL },
-    { NULL }
-};
-
-static void
-on_message_received (SoupWebsocketConnection *socket_connection,
-                     gint type, GBytes *message,
-                     gpointer data)
-{
-    ConnectionContext *ctx = (ConnectionContext*) data;
-
-    debug_printf (1, "<- ");
-
-    if (ctx && ctx->method)
-        ctx->method (socket_connection, type, message, ctx->data);
-}
-
-static void
-on_connection_closed (SoupWebsocketConnection *socket_connection,
-                      gpointer data)
-{
-    ConnectionContext *ctx = (ConnectionContext*) data;
-
-    debug_printf (1, "\nConnection closed\n");
-
-    g_free (ctx);
-
-    g_object_unref (socket_connection);
-    g_main_loop_quit (loop);
-}
-
-static void
-on_connect (GObject *session,
-            GAsyncResult *res,
-            gpointer ctx)
-{
-    SoupWebsocketConnection *socket_connection = soup_session_websocket_connect_finish (SOUP_SESSION(session), res, NULL);
-    if (!socket_connection) {
-        g_free (ctx);
-        return;
-    }
-
-    /* The performance tests increase the size of the payload up to 16 MB, let's disable
-       the limit to see what happens. */
-    soup_websocket_connection_set_max_incoming_payload_size (socket_connection, 0);
-
-    g_signal_connect (socket_connection, "message", G_CALLBACK(on_message_received), ctx);
-    g_signal_connect (socket_connection, "closed", G_CALLBACK(on_connection_closed), ctx);
-}
-
-static void
-connect_and_run (SoupSession *session, char *path, ConnectionFunc method, gpointer data)
-{
-    char *uri = g_strconcat (address, path, NULL);
-    SoupMessage *message = soup_message_new (SOUP_METHOD_GET, uri);
-    ConnectionContext *ctx  = g_new0 (ConnectionContext, 1);
-
-    ctx->method = method;
-    ctx->data = data;
-
-    debug_printf (1, "About to connect to %s\n", uri);
-    soup_session_websocket_connect_async (session, message, NULL, NULL, G_PRIORITY_DEFAULT, NULL, on_connect, ctx);
-
-    g_object_unref (message);
-    g_free (uri);
-
-    g_main_loop_run (loop);
-}
-
-static void
-test_case_message_received (SoupWebsocketConnection *socket_connection,
-                            gint type,
-                            GBytes *message,
-                            gpointer data)
-{
-    /* Cannot send messages if we're not in an open state. */
-    if (soup_websocket_connection_get_state (socket_connection) != SOUP_WEBSOCKET_STATE_OPEN)
-        return;
-
-    debug_printf (1, "-> ");
-
-    soup_websocket_connection_send_message (socket_connection, type, message);
-}
-
-static void
-test_case (gconstpointer data)
-{
-    TestBundle *bundle = (TestBundle *) data;
-
-    SoupSession *session = bundle->session;
-    char *path = bundle->path;
-    connect_and_run (session, path, test_case_message_received, bundle);
-
-    g_free(path);
-}
-
-static void
-run_case (SoupSession *session, unsigned int num_test_case)
-{
-    char *path = g_strdup_printf ("/runCase?case=%u&agent=%s", num_test_case, agent);
-    TestBundle *bundle = g_new0 (TestBundle, 1);
-    bundle->session = session;
-    bundle->num_test_case = num_test_case;
-    bundle->path = g_strdup(path);
-    g_test_add_data_func (path, bundle, test_case);
-    g_free (path);
-}
-
-static void
-run_test_cases (SoupSession *session, unsigned int num_cases)
-{
-    int i;
-    for (i = 0; i < num_cases; i++)
-        run_case (session, i + 1);
-}
-
-static void
-run_all_test_cases (SoupSession *session)
-{
-    run_test_cases (session, total_num_cases);
-}
-
-static void
-got_case_count (SoupWebsocketConnection *socket_connection,
-                gint type,
-                GBytes *message,
-                gpointer data)
-{
-    total_num_cases = g_ascii_strtoull (g_bytes_get_data (message, NULL), NULL, 10);
-
-    debug_printf (1, "Total number of cases: %u\n", total_num_cases);
-}
-
-static void
-get_case_count (SoupSession *session)
-{
-    connect_and_run (session, "/getCaseCount", got_case_count, NULL);
-}
-
-static void
-update_reports (SoupSession *session)
-{
-    char *path = g_strdup_printf ("/updateReports?agent=%s", agent);
-    debug_printf (1, "Updating reports..\n");
-    connect_and_run (session, path, NULL, NULL);
-    g_free (path);
-}
-
-static gboolean
-autobahn_server (const char *action)
-{
-    GPtrArray *argv;
-    char *cwd;
-
-    cwd = g_get_current_dir();
-
-    char *dirname = g_path_get_dirname (__FILE__);
-    char *path = g_build_filename(dirname, "autobahn-server.sh", NULL);
-
-    argv = g_ptr_array_new ();
-    g_ptr_array_add (argv, path);
-    g_ptr_array_add (argv, (char *) action);
-    g_ptr_array_add (argv, NULL);
-
-    gboolean ret = g_spawn_async (cwd, (char **) argv->pdata, NULL, 0, NULL, NULL,
-                          NULL, NULL);
-    g_free (path);
-
-    return ret;
-}
-
-static void
-start_autobahn (void)
-{
-    gboolean code = autobahn_server ("--start");
-    if (code == FALSE) {
-        debug_printf(1, "Could not start Autobahn server\n");
-        exit(1);
-    }
-}
-
-static void
-stop_autobahn (void)
-{
-    autobahn_server ("--stop");
-}
-
-int main (int argc, char* argv[])
-{
-    int ret = 0;
-
-    GOptionContext *context;
-    GError *error = NULL;
-    SoupSession *session;
-
-    debug_printf (1, "Starting Autobahn server\n");
-    start_autobahn ();
-    sleep (2);
-
-    context = g_option_context_new ("- libsoup test runner for Autobahn WebSocket client tests.");
-    g_option_context_add_main_entries (context, entries, NULL);
-
-    if (!g_option_context_parse (context, &argc, &argv, &error)) {
-        g_warning ("Option parsing failed: %s\n", error->message);
-        g_error_free (error);
-        g_option_context_free (context);
-        exit (1);
-    }
-    g_option_context_free (context);
-
-    if (option_run_test >= 0 || option_number_of_tests)
-        option_run_all = FALSE;
-
-    session = soup_session_new ();
-    soup_session_add_feature_by_type (session, SOUP_TYPE_WEBSOCKET_EXTENSION_MANAGER);
-    loop = g_main_loop_new (g_main_context_default (), FALSE);
-
-    if (!(option_run_all || option_number_of_tests || option_update_report || option_run_test > 0))
-        option_run_all = TRUE;
-
-    if (option_run_all || option_number_of_tests)
-        get_case_count (session);
-
-    if (option_run_test >= 0 || option_run_all) {
-        test_init (argc, argv, NULL);
-        if (option_run_test >= 0) {
-            run_case (session, option_run_test);
-        } else {
-            run_all_test_cases (session);
-        }
-        ret = g_test_run();
-        test_cleanup();
-    }
-
-    if (option_update_report)
-        update_reports (session);
-
-    g_object_unref (session);
-
-    stop_autobahn ();
-
-    return ret;
-}
diff --git a/tests/meson.build b/tests/meson.build
index c847a254..0892cccf 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -137,6 +137,10 @@ env.set('MALLOC_CHECK_', '2')
 # This is set by Meson if empty
 env.set('MALLOC_PERTURB_', '')
 
+if have_autobahn
+   subdir('autobahn')
+endif
+
 foreach test: tests
   test_name = '@0@-test'.format(test[0])