*) mod_http: genereate HEADERS buckets for trailers

     mod_proxy: forward trailers on chunked request encoding
     test: add http/1.x test cases in pytest



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1899552 13f79535-47bb-0310-9956-ffa450edef68

19 files changed, 810 insertions, 48 deletions

diff --git a/modules/http/http_filters.c b/modules/http/http_filters.c
index 9bcf2297d0..0a9ff425ea 100644
--- a/modules/http/http_filters.c
+++ b/modules/http/http_filters.c
@@ -1312,6 +1312,15 @@ AP_DECLARE_NONSTD(int) ap_send_http_trace(request_rec *r)
     return DONE;
 }
 
+static apr_bucket *create_trailers_bucket(request_rec *r, apr_bucket_alloc_t *bucket_alloc)
+{
+    if (r->trailers_out && !apr_is_empty_table(r->trailers_out)) {
+        ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, r, "sending trailers");
+        return ap_bucket_headers_create(r->trailers_out, r->pool, bucket_alloc);
+    }
+    return NULL;
+}
+
 typedef struct header_filter_ctx {
     int headers_sent;
 } header_filter_ctx;
@@ -1323,7 +1332,7 @@ AP_CORE_DECLARE_NONSTD(apr_status_t) ap_http_header_filter(ap_filter_t *f,
     conn_rec *c = r->connection;
     int header_only = (r->header_only || AP_STATUS_IS_HEADER_ONLY(r->status));
     const char *protocol = NULL;
-    apr_bucket *e;
+    apr_bucket *e, *eos = NULL;
     apr_bucket_brigade *b2;
     header_struct h;
     header_filter_ctx *ctx = f->ctx;
@@ -1364,6 +1373,10 @@ AP_CORE_DECLARE_NONSTD(apr_status_t) ap_http_header_filter(ap_filter_t *f,
             eb = e->data;
             continue;
         }
+        if (APR_BUCKET_IS_EOS(e)) {
+            if (!eos) eos = e;
+            continue;
+        }
         /*
          * If we see an EOC bucket it is a signal that we should get out
          * of the way doing nothing.
@@ -1416,6 +1429,22 @@ AP_CORE_DECLARE_NONSTD(apr_status_t) ap_http_header_filter(ap_filter_t *f,
         goto out;
     }
 
+    if (eos) {
+        /* on having seen EOS and added possible trailers, we
+         * can remove this filter.
+         */
+        e = create_trailers_bucket(r, b->bucket_alloc);
+        if (e) {
+            APR_BUCKET_INSERT_BEFORE(eos, e);
+        }
+        ap_remove_output_filter(f);
+    }
+
+    if (ctx->headers_sent) {
+        /* we did already the stuff below, just pass on */
+        return ap_pass_brigade(f->next, b);
+    }
+
     /*
      * Now that we are ready to send a response, we need to combine the two
      * header field tables into a single table.  If we don't do this, our
@@ -1545,11 +1574,6 @@ AP_CORE_DECLARE_NONSTD(apr_status_t) ap_http_header_filter(ap_filter_t *f,
         ap_add_output_filter("CHUNK", NULL, r, r->connection);
     }
 
-    /* Don't remove this filter until after we have added the CHUNK filter.
-     * Otherwise, f->next won't be the CHUNK filter and thus the first
-     * brigade won't be chunked properly.
-     */
-    ap_remove_output_filter(f);
     rv = ap_pass_brigade(f->next, b);
 out:
     if (recursive_error) {
diff --git a/modules/proxy/mod_proxy_http.c b/modules/proxy/mod_proxy_http.c
index b617c8e728..4930eda08e 100644
--- a/modules/proxy/mod_proxy_http.c
+++ b/modules/proxy/mod_proxy_http.c
@@ -455,14 +455,7 @@ static int stream_reqbody(proxy_http_req_t *req)
                     APR_BRIGADE_INSERT_TAIL(input_brigade, e);
                 }
                 if (seen_eos) {
-                    /*
-                     * Append the tailing 0-size chunk
-                     */
-                    e = apr_bucket_immortal_create(ZERO_ASCII CRLF_ASCII
-                                                   /* <trailers> */
-                                                   CRLF_ASCII,
-                                                   5, bucket_alloc);
-                    APR_BRIGADE_INSERT_TAIL(input_brigade, e);
+                    ap_h1_add_end_chunk(input_brigade, NULL, r, r->trailers_in);
                 }
             }
             else if (rb_method == RB_STREAM_CL
diff --git a/test/modules/http1/__init__.py b/test/modules/http1/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/test/modules/http1/__init__.py
diff --git a/test/modules/http1/conftest.py b/test/modules/http1/conftest.py
new file mode 100644
index 0000000000..0ebb439129
--- /dev/null
+++ b/test/modules/http1/conftest.py
@@ -0,0 +1,47 @@
+import logging
+import os
+
+import pytest
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
+
+from .env import H1TestEnv
+
+
+def pytest_report_header(config, startdir):
+    env = H1TestEnv()
+    return f"mod_http [apache: {env.get_httpd_version()}, mpm: {env.mpm_module}, {env.prefix}]"
+
+
+def pytest_generate_tests(metafunc):
+    if "repeat" in metafunc.fixturenames:
+        count = int(metafunc.config.getoption("repeat"))
+        metafunc.fixturenames.append('tmp_ct')
+        metafunc.parametrize('repeat', range(count))
+
+
+@pytest.fixture(scope="package")
+def env(pytestconfig) -> H1TestEnv:
+    level = logging.INFO
+    console = logging.StreamHandler()
+    console.setLevel(level)
+    console.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+    logging.getLogger('').addHandler(console)
+    logging.getLogger('').setLevel(level=level)
+    env = H1TestEnv(pytestconfig=pytestconfig)
+    env.setup_httpd()
+    env.apache_access_log_clear()
+    env.httpd_error_log.clear_log()
+    return env
+
+
+@pytest.fixture(autouse=True, scope="package")
+def _session_scope(env):
+    yield
+    assert env.apache_stop() == 0
+    errors, warnings = env.httpd_error_log.get_missed()
+    assert (len(errors), len(warnings)) == (0, 0),\
+            f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\
+            "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings))
+
diff --git a/test/modules/http1/env.py b/test/modules/http1/env.py
new file mode 100644
index 0000000000..55dfbe2683
--- /dev/null
+++ b/test/modules/http1/env.py
@@ -0,0 +1,88 @@
+import inspect
+import logging
+import os
+import re
+import subprocess
+from typing import Dict, Any
+
+from pyhttpd.certs import CertificateSpec
+from pyhttpd.conf import HttpdConf
+from pyhttpd.env import HttpdTestEnv, HttpdTestSetup
+
+log = logging.getLogger(__name__)
+
+
+class H1TestSetup(HttpdTestSetup):
+
+    def __init__(self, env: 'HttpdTestEnv'):
+        super().__init__(env=env)
+        self.add_source_dir(os.path.dirname(inspect.getfile(H1TestSetup)))
+        self.add_modules(["cgid", "autoindex", "ssl"])
+
+    def make(self):
+        super().make()
+        self._add_h1test()
+        self._setup_data_1k_1m()
+
+    def _add_h1test(self):
+        local_dir = os.path.dirname(inspect.getfile(H1TestSetup))
+        p = subprocess.run([self.env.apxs, '-c', 'mod_h1test.c'],
+                           capture_output=True,
+                           cwd=os.path.join(local_dir, 'mod_h1test'))
+        rv = p.returncode
+        if rv != 0:
+            log.error(f"compiling md_h1test failed: {p.stderr}")
+            raise Exception(f"compiling md_h1test failed: {p.stderr}")
+
+        modules_conf = os.path.join(self.env.server_dir, 'conf/modules.conf')
+        with open(modules_conf, 'a') as fd:
+            # load our test module which is not installed
+            fd.write(f"LoadModule h1test_module   \"{local_dir}/mod_h1test/.libs/mod_h1test.so\"\n")
+
+    def _setup_data_1k_1m(self):
+        s90 = "01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678\n"
+        with open(os.path.join(self.env.gen_dir, "data-1k"), 'w') as f:
+            for i in range(10):
+                f.write(f"{i:09d}-{s90}")
+        with open(os.path.join(self.env.gen_dir, "data-10k"), 'w') as f:
+            for i in range(100):
+                f.write(f"{i:09d}-{s90}")
+        with open(os.path.join(self.env.gen_dir, "data-100k"), 'w') as f:
+            for i in range(1000):
+                f.write(f"{i:09d}-{s90}")
+        with open(os.path.join(self.env.gen_dir, "data-1m"), 'w') as f:
+            for i in range(10000):
+                f.write(f"{i:09d}-{s90}")
+
+
+class H1TestEnv(HttpdTestEnv):
+
+    def __init__(self, pytestconfig=None):
+        super().__init__(pytestconfig=pytestconfig)
+        self.add_httpd_log_modules(["http", "core"])
+
+        self.httpd_error_log.set_ignored_lognos([
+            'AH00135', # unsafe/strict tests send invalid methods
+        ])
+        self.httpd_error_log.add_ignored_patterns([
+        ])
+
+    def setup_httpd(self, setup: HttpdTestSetup = None):
+        super().setup_httpd(setup=H1TestSetup(env=self))
+
+
+class H1Conf(HttpdConf):
+
+    def __init__(self, env: HttpdTestEnv, extras: Dict[str, Any] = None):
+        super().__init__(env=env, extras=HttpdConf.merge_extras(extras, {
+            "base": [
+                "LogLevel http:trace4",
+            ],
+            f"cgi.{env.http_tld}": [
+                "SSLOptions +StdEnvVars",
+                "AddHandler cgi-script .py",
+                "<Location \"/h1test/echo\">",
+                "    SetHandler h1test-echo",
+                "</Location>",
+            ]
+        }))
diff --git a/test/modules/http1/htdocs/cgi/files/empty.txt b/test/modules/http1/htdocs/cgi/files/empty.txt
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/test/modules/http1/htdocs/cgi/files/empty.txt
diff --git a/test/modules/http1/htdocs/cgi/hello.py b/test/modules/http1/htdocs/cgi/hello.py
new file mode 100755
index 0000000000..191acb2fed
--- /dev/null
+++ b/test/modules/http1/htdocs/cgi/hello.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+import os
+
+print("Content-Type: application/json")
+print()
+print("{")
+print("  \"https\" : \"%s\"," % (os.getenv('HTTPS', '')))
+print("  \"host\" : \"%s\"," % (os.getenv('SERVER_NAME', '')))
+print("  \"protocol\" : \"%s\"," % (os.getenv('SERVER_PROTOCOL', '')))
+print("  \"ssl_protocol\" : \"%s\"," % (os.getenv('SSL_PROTOCOL', '')))
+print("  \"h2\" : \"%s\"," % (os.getenv('HTTP2', '')))
+print("  \"h2push\" : \"%s\"" % (os.getenv('H2PUSH', '')))
+print("}")
+
diff --git a/test/modules/http1/htdocs/cgi/upload.py b/test/modules/http1/htdocs/cgi/upload.py
new file mode 100755
index 0000000000..7f599d27ef
--- /dev/null
+++ b/test/modules/http1/htdocs/cgi/upload.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+import cgi, os
+import cgitb
+
+cgitb.enable()
+
+status = '200 Ok'
+
+try:  # Windows needs stdio set for binary mode.
+    import msvcrt
+
+    msvcrt.setmode(0, os.O_BINARY)  # stdin  = 0
+    msvcrt.setmode(1, os.O_BINARY)  # stdout = 1
+except ImportError:
+    pass
+
+form = cgi.FieldStorage()
+
+# Test if the file was uploaded
+if 'file' in form:
+    fileitem = form['file']
+    # strip leading path from file name to avoid directory traversal attacks
+    fn = os.path.basename(fileitem.filename)
+    f = open(('%s/files/%s' % (os.environ["DOCUMENT_ROOT"], fn)), 'wb');
+    f.write(fileitem.file.read())
+    f.close()
+    message = "The file %s was uploaded successfully" % (fn)
+    print("Status: 201 Created")
+    print("Content-Type: text/html")
+    print("Location: %s://%s/files/%s" % (os.environ["REQUEST_SCHEME"], os.environ["HTTP_HOST"], fn))
+    print("")
+    print("<html><body><p>%s</p></body></html>" % (message))
+
+elif 'remove' in form:
+    remove = form['remove'].value
+    try:
+        fn = os.path.basename(remove)
+        os.remove('./files/' + fn)
+        message = 'The file "' + fn + '" was removed successfully'
+    except OSError as e:
+        message = 'Error removing ' + fn + ': ' + e.strerror
+        status = '404 File Not Found'
+    print("Status: %s" % (status))
+    print("""
+Content-Type: text/html
+
+<html><body>
+<p>%s</p>
+</body></html>""" % (message))
+
+else:
+    message = '''\
+        Upload File<form method="POST" enctype="multipart/form-data">
+        <input type="file" name="file">
+        <button type="submit">Upload</button></form>
+        '''
+    print("Status: %s" % (status))
+    print("""\
+Content-Type: text/html
+
+<html><body>
+<p>%s</p>
+</body></html>""" % (message))
+
diff --git a/test/modules/http1/mod_h1test/mod_h1test.c b/test/modules/http1/mod_h1test/mod_h1test.c
new file mode 100644
index 0000000000..cbd87b5b6c
--- /dev/null
+++ b/test/modules/http1/mod_h1test/mod_h1test.c
@@ -0,0 +1,129 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <apr_optional.h>
+#include <apr_optional_hooks.h>
+#include <apr_strings.h>
+#include <apr_cstr.h>
+#include <apr_time.h>
+#include <apr_want.h>
+
+#include <httpd.h>
+#include <http_protocol.h>
+#include <http_request.h>
+#include <http_log.h>
+
+static void h1test_hooks(apr_pool_t *pool);
+
+AP_DECLARE_MODULE(h1test) = {
+    STANDARD20_MODULE_STUFF,
+    NULL, /* func to create per dir config */
+    NULL,  /* func to merge per dir config */
+    NULL, /* func to create per server config */
+    NULL,  /* func to merge per server config */
+    NULL,              /* command handlers */
+    h1test_hooks,
+#if defined(AP_MODULE_FLAG_NONE)
+    AP_MODULE_FLAG_ALWAYS_MERGE
+#endif
+};
+
+
+static int h1test_echo_handler(request_rec *r)
+{
+    conn_rec *c = r->connection;
+    apr_bucket_brigade *bb;
+    apr_bucket *b;
+    apr_status_t rv;
+    char buffer[8192];
+    const char *ct;
+    long l;
+
+    if (strcmp(r->handler, "h1test-echo")) {
+        return DECLINED;
+    }
+    if (r->method_number != M_GET && r->method_number != M_POST) {
+        return DECLINED;
+    }
+
+    ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "echo_handler: processing request");
+    r->status = 200;
+    r->clength = -1;
+    r->chunked = 1;
+    ct = apr_table_get(r->headers_in, "content-type");
+    ap_set_content_type(r, ct? ct : "application/octet-stream");
+
+    bb = apr_brigade_create(r->pool, c->bucket_alloc);
+    /* copy any request body into the response */
+    if ((rv = ap_setup_client_block(r, REQUEST_CHUNKED_DECHUNK))) goto cleanup;
+    if (ap_should_client_block(r)) {
+        while (0 < (l = ap_get_client_block(r, &buffer[0], sizeof(buffer)))) {
+            ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r,
+                          "echo_handler: copying %ld bytes from request body", l);
+            rv = apr_brigade_write(bb, NULL, NULL, buffer, l);
+            if (APR_SUCCESS != rv) goto cleanup;
+            rv = ap_pass_brigade(r->output_filters, bb);
+            if (APR_SUCCESS != rv) goto cleanup;
+            ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r,
+                          "echo_handler: passed %ld bytes from request body", l);
+        }
+    }
+    /* we are done */
+    b = apr_bucket_eos_create(c->bucket_alloc);
+    APR_BRIGADE_INSERT_TAIL(bb, b);
+    ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "echo_handler: request read");
+
+    if (r->trailers_in && !apr_is_empty_table(r->trailers_in)) {
+        ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, r,
+                      "echo_handler: seeing incoming trailers");
+        apr_table_setn(r->trailers_out, "h1test-trailers-in",
+                       apr_itoa(r->pool, 1));
+    }
+    if (apr_table_get(r->headers_in, "Add-Trailer")) {
+        ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, r,
+                      "echo_handler: seeing incoming Add-Trailer header");
+        apr_table_setn(r->trailers_out, "h1test-add-trailer",
+                       apr_table_get(r->headers_in, "Add-Trailer"));
+    }
+
+    rv = ap_pass_brigade(r->output_filters, bb);
+
+cleanup:
+    if (rv == APR_SUCCESS
+        || r->status != HTTP_OK
+        || c->aborted) {
+        ap_log_rerror(APLOG_MARK, APLOG_TRACE1, rv, r, "echo_handler: request handled");
+        return OK;
+    }
+    else {
+        /* no way to know what type of error occurred */
+        ap_log_rerror(APLOG_MARK, APLOG_TRACE1, rv, r, "h1test_echo_handler failed");
+        return AP_FILTER_ERROR;
+    }
+    return DECLINED;
+}
+
+
+/* Install this module into the apache2 infrastructure.
+ */
+static void h1test_hooks(apr_pool_t *pool)
+{
+    ap_log_perror(APLOG_MARK, APLOG_TRACE1, 0, pool, "installing hooks and handlers");
+
+    /* test h1 handlers */
+    ap_hook_handler(h1test_echo_handler, NULL, NULL, APR_HOOK_MIDDLE);
+}
+
diff --git a/test/modules/http1/mod_h1test/mod_h1test.slo b/test/modules/http1/mod_h1test/mod_h1test.slo
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/test/modules/http1/mod_h1test/mod_h1test.slo
diff --git a/test/modules/http1/test_001_alive.py b/test/modules/http1/test_001_alive.py
new file mode 100644
index 0000000000..0a1de1dc6b
--- /dev/null
+++ b/test/modules/http1/test_001_alive.py
@@ -0,0 +1,20 @@
+import pytest
+
+from .env import H1Conf
+
+
+class TestBasicAlive:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        H1Conf(env).add_vhost_test1().install()
+        assert env.apache_restart() == 0
+
+    # we expect to see the document from the generic server
+    def test_h1_001_01(self, env):
+        url = env.mkurl("https", "test1", "/alive.json")
+        r = env.curl_get(url, 5)
+        assert r.exit_code == 0, r.stderr + r.stdout
+        assert r.response["json"]
+        assert r.response["json"]["alive"] is True
+        assert r.response["json"]["host"] == "test1"
diff --git a/test/modules/http1/test_003_get.py b/test/modules/http1/test_003_get.py
new file mode 100644
index 0000000000..21fd4ef5bf
--- /dev/null
+++ b/test/modules/http1/test_003_get.py
@@ -0,0 +1,28 @@
+import re
+import socket
+
+import pytest
+
+from .env import H1Conf
+
+
+class TestGet:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        H1Conf(env).add_vhost_cgi(
+            proxy_self=True
+        ).add_vhost_test1(
+            proxy_self=True
+        ).install()
+        assert env.apache_restart() == 0
+
+    # check SSL environment variables from CGI script
+    def test_h1_003_01(self, env):
+        url = env.mkurl("https", "cgi", "/hello.py")
+        r = env.curl_get(url)
+        assert r.response["status"] == 200
+        assert r.response["json"]["protocol"] == "HTTP/1.1"
+        assert r.response["json"]["https"] == "on"
+        tls_version = r.response["json"]["ssl_protocol"]
+        assert tls_version in ["TLSv1.2", "TLSv1.3"]
diff --git a/test/modules/http1/test_004_post.py b/test/modules/http1/test_004_post.py
new file mode 100644
index 0000000000..d4d5edcbf5
--- /dev/null
+++ b/test/modules/http1/test_004_post.py
@@ -0,0 +1,54 @@
+import difflib
+import email.parser
+import inspect
+import json
+import os
+import re
+import sys
+
+import pytest
+
+from .env import H1Conf
+
+
+class TestPost:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        TestPost._local_dir = os.path.dirname(inspect.getfile(TestPost))
+        H1Conf(env).add_vhost_cgi().install()
+        assert env.apache_restart() == 0
+
+    def local_src(self, fname):
+        return os.path.join(TestPost._local_dir, fname)
+
+    # upload and GET again using curl, compare to original content
+    def curl_upload_and_verify(self, env, fname, options=None):
+        url = env.mkurl("https", "cgi", "/upload.py")
+        fpath = os.path.join(env.gen_dir, fname)
+        r = env.curl_upload(url, fpath, options=options)
+        assert r.exit_code == 0, f"{r}"
+        assert 200 <= r.response["status"] < 300
+
+        r2 = env.curl_get(r.response["header"]["location"])
+        assert r2.exit_code == 0
+        assert r2.response["status"] == 200
+        with open(self.local_src(fpath), mode='rb') as file:
+            src = file.read()
+        assert src == r2.response["body"]
+        return r
+
+    def test_h1_004_01(self, env):
+        self.curl_upload_and_verify(env, "data-1k", ["-vvv"])
+
+    def test_h1_004_02(self, env):
+        self.curl_upload_and_verify(env, "data-10k", [])
+
+    def test_h1_004_03(self, env):
+        self.curl_upload_and_verify(env, "data-100k", [])
+
+    def test_h1_004_04(self, env):
+        self.curl_upload_and_verify(env, "data-1m", [])
+
+    def test_h1_004_05(self, env):
+        r = self.curl_upload_and_verify(env, "data-1k", ["-vvv", "-H", "Expect: 100-continue"])
diff --git a/test/modules/http1/test_005_trailers.py b/test/modules/http1/test_005_trailers.py
new file mode 100644
index 0000000000..36439330ce
--- /dev/null
+++ b/test/modules/http1/test_005_trailers.py
@@ -0,0 +1,38 @@
+import os
+import pytest
+
+from .env import H1Conf
+
+
+# The trailer tests depend on "nghttp" as no other client seems to be able to send those
+# rare things.
+class TestTrailers:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        H1Conf(env).add_vhost_cgi(proxy_self=True).install()
+        assert env.apache_restart() == 0
+
+    # check that we get a trailer out when telling the handler to add one
+    def test_h1_005_01(self, env):
+        url = env.mkurl("https", "cgi", "/h1test/echo")
+        host = f"cgi.{env.http_tld}"
+        fpath = os.path.join(env.gen_dir, "data-1k")
+        r = env.curl_upload(url, fpath, options=["--header", "Add-Trailer: 005_01"])
+        assert r.exit_code == 0, f"{r}"
+        assert 200 <= r.response["status"] < 300
+        assert r.response["trailer"], f"no trailers received: {r}"
+        assert "h1test-add-trailer" in r.response["trailer"]
+        assert r.response["trailer"]["h1test-add-trailer"] == "005_01"
+
+    # check that we get out trailers through the proxy
+    def test_h1_005_02(self, env):
+        url = env.mkurl("https", "cgi", "/proxy/h1test/echo")
+        host = f"cgi.{env.http_tld}"
+        fpath = os.path.join(env.gen_dir, "data-1k")
+        r = env.curl_upload(url, fpath, options=["--header", "Add-Trailer: 005_01"])
+        assert r.exit_code == 0, f"{r}"
+        assert 200 <= r.response["status"] < 300
+        assert r.response["trailer"], f"no trailers received: {r}"
+        assert "h1test-add-trailer" in r.response["trailer"]
+        assert r.response["trailer"]["h1test-add-trailer"] == "005_01"
diff --git a/test/modules/http1/test_006_unsafe.py b/test/modules/http1/test_006_unsafe.py
new file mode 100644
index 0000000000..b6882292c2
--- /dev/null
+++ b/test/modules/http1/test_006_unsafe.py
@@ -0,0 +1,132 @@
+import re
+import socket
+from typing import Optional
+
+import pytest
+
+from .env import H1Conf
+
+class TestRequestUnsafe:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        conf = H1Conf(env)
+        conf.add([
+            "HttpProtocolOptions Unsafe",
+        ])
+        conf.install()
+        assert env.apache_restart() == 0
+
+    # unsafe tests from t/apache/http_strict.t
+    # possible expected results:
+    #   0:       any HTTP error
+    #   1:       any HTTP success
+    #   200-500: specific HTTP status code
+    #   None:   HTTPD should drop connection without error message
+    @pytest.mark.parametrize(["intext", "status"], [
+        ["GET / HTTP/1.0\r\n\r\n", 1],
+        ["GET / HTTP/1.0\n\n", 1],
+        ["get / HTTP/1.0\r\n\r\n", 501],
+        ["G ET / HTTP/1.0\r\n\r\n", 400],
+        ["G\0ET / HTTP/1.0\r\n\r\n", 400],
+        ["G/T / HTTP/1.0\r\n\r\n", 501],
+        ["GET /\0 HTTP/1.0\r\n\r\n", 400],
+        ["GET / HTTP/1.0\0\r\n\r\n", 400],
+        ["GET\f/ HTTP/1.0\r\n\r\n", 400],
+        ["GET\r/ HTTP/1.0\r\n\r\n", 400],
+        ["GET\t/ HTTP/1.0\r\n\r\n", 400],
+        ["GET / HTT/1.0\r\n\r\n", 0],
+        ["GET / HTTP/1.0\r\nHost: localhost\r\n\r\n", 1],
+        ["GET / HTTP/2.0\r\nHost: localhost\r\n\r\n", 1],
+        ["GET / HTTP/1.2\r\nHost: localhost\r\n\r\n", 1],
+        ["GET / HTTP/1.11\r\nHost: localhost\r\n\r\n", 400],
+        ["GET / HTTP/10.0\r\nHost: localhost\r\n\r\n", 400],
+        ["GET / HTTP/1.0  \r\nHost: localhost\r\n\r\n", 200],
+        ["GET / HTTP/1.0 x\r\nHost: localhost\r\n\r\n", 400],
+        ["GET / HTTP/\r\nHost: localhost\r\n\r\n", 0],
+        ["GET / HTTP/0.9\r\n\r\n", 0],
+        ["GET / HTTP/0.8\r\n\r\n", 0],
+        ["GET /\x01 HTTP/1.0\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nFoo: bar\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nFoo:bar\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nFoo: b\0ar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nFoo: b\x01ar\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nFoo\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nFoo bar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\n: bar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nX: bar\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nFoo bar:bash\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nFoo :bar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\n Foo:bar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nF\x01o: bar\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nF\ro: bar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nF\to: bar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nFo: b\tar\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nFo: bar\r\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\r", None],
+        ["GET /\r\n", 90],
+        ["GET /#frag HTTP/1.0\r\n", 400],
+        ["GET / HTTP/1.0\r\nHost: localhost\r\nHost: localhost\r\n\r\n", 200],
+        ["GET http://017700000001/ HTTP/1.0\r\n\r\n", 200],
+        ["GET http://0x7f.1/ HTTP/1.0\r\n\r\n", 200],
+        ["GET http://127.0.0.1/ HTTP/1.0\r\n\r\n", 200],
+        ["GET http://127.01.0.1/ HTTP/1.0\r\n\r\n", 200],
+        ["GET http://%3127.0.0.1/ HTTP/1.0\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nHost: localhost:80\r\nHost: localhost:80\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nHost: localhost:80 x\r\n\r", 400],
+        ["GET http://localhost:80/ HTTP/1.0\r\n\r\n", 200],
+        ["GET http://localhost:80x/ HTTP/1.0\r\n\r\n", 400],
+        ["GET http://localhost:80:80/ HTTP/1.0\r\n\r\n", 400],
+        ["GET http://localhost::80/ HTTP/1.0\r\n\r\n", 400],
+        ["GET http://foo@localhost:80/ HTTP/1.0\r\n\r\n", 200],
+        ["GET http://[::1]/ HTTP/1.0\r\n\r\n", 1],
+        ["GET http://[::1:2]/ HTTP/1.0\r\n\r\n", 1],
+        ["GET http://[4712::abcd]/ HTTP/1.0\r\n\r\n", 1],
+        ["GET http://[4712::abcd:1]/ HTTP/1.0\r\n\r\n", 1],
+        ["GET http://[4712::abcd::]/ HTTP/1.0\r\n\r\n", 400],
+        ["GET http://[4712:abcd::]/ HTTP/1.0\r\n\r\n", 1],
+        ["GET http://[4712::abcd]:8000/ HTTP/1.0\r\n\r\n", 1],
+        ["GET http://4713::abcd:8001/ HTTP/1.0\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nHost: [::1]\r\n\r\n", 1],
+        ["GET / HTTP/1.0\r\nHost: [::1:2]\r\n\r\n", 1],
+        ["GET / HTTP/1.0\r\nHost: [4711::abcd]\r\n\r\n", 1],
+        ["GET / HTTP/1.0\r\nHost: [4711::abcd:1]\r\n\r\n", 1],
+        ["GET / HTTP/1.0\r\nHost: [4711:abcd::]\r\n\r\n", 1],
+        ["GET / HTTP/1.0\r\nHost: [4711::abcd]:8000\r\n\r\n", 1],
+        ["GET / HTTP/1.0\r\nHost: 4714::abcd:8001\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nHost: abc\xa0\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nHost: abc\\foo\r\n\r\n", 400],
+        ["GET http://foo/ HTTP/1.0\r\nHost: bar\r\n\r\n", 200],
+        ["GET http://foo:81/ HTTP/1.0\r\nHost: bar\r\n\r\n", 200],
+        ["GET http://[::1]:81/ HTTP/1.0\r\nHost: bar\r\n\r\n", 200],
+        ["GET http://10.0.0.1:81/ HTTP/1.0\r\nHost: bar\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nHost: foo-bar.example.com\r\n\r\n", 200],
+        ["GET / HTTP/1.0\r\nHost: foo_bar.example.com\r\n\r\n", 200],
+        ["GET http://foo_bar/ HTTP/1.0\r\n\r\n", 200],
+    ])
+    def test_h1_006_01(self, env, intext, status: Optional[int]):
+        with socket.create_connection(('localhost', int(env.http_port))) as sock:
+            # on some OS, the server does not see our connection until there is
+            # something incoming
+            sock.sendall(intext.encode())
+            sock.shutdown(socket.SHUT_WR)
+            buff = sock.recv(1024)
+            msg = buff.decode()
+            if status is None:
+                assert len(msg) == 0, f"unexpected answer: {msg}"
+            else:
+                assert len(msg) > 0, "no answer from server"
+                rlines = msg.splitlines()
+                response = rlines[0]
+                m = re.match(r'^HTTP/1.1 (\d+)\s+(\S+)', response)
+                assert m or status == 90, f"unrecognized response: {rlines}"
+                if status == 1:
+                    assert int(m.group(1)) >= 200
+                elif status == 90:
+                    # headerless 0.9 response, yuk
+                    assert len(rlines) >= 1, f"{rlines}"
+                elif status > 0:
+                    assert int(m.group(1)) == status, f"{rlines}"
+                else:
+                    assert int(m.group(1)) >= 400, f"{rlines}"
+
diff --git a/test/modules/http1/test_007_strict.py b/test/modules/http1/test_007_strict.py
new file mode 100644
index 0000000000..4649b01123
--- /dev/null
+++ b/test/modules/http1/test_007_strict.py
@@ -0,0 +1,69 @@
+import re
+import socket
+from typing import Optional
+
+import pytest
+
+from .env import H1Conf
+
+
+class TestRequestStrict:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        conf = H1Conf(env)
+        conf.add([
+            "HttpProtocolOptions Strict",
+        ])
+        conf.install()
+        assert env.apache_restart() == 0
+
+    # strict tests from t/apache/http_strict.t
+    # possible expected results:
+    #   0:       any HTTP error
+    #   1:       any HTTP success
+    #   200-500: specific HTTP status code
+    #   undef:   HTTPD should drop connection without error message
+    @pytest.mark.parametrize(["intext", "status"], [
+        ["GET / HTTP/1.0\n\n", 400],
+        ["G/T / HTTP/1.0\r\n\r\n", 400],
+        ["GET / HTTP/1.0  \r\nHost: localhost\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nFoo: b\x01ar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nF\x01o: bar\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\r", None],
+        ["GET / HTTP/1.0\r\nHost: localhost\r\nHost: localhost\r\n\r\n", 400],
+        ["GET http://017700000001/ HTTP/1.0\r\n\r\n", 400],
+        ["GET http://0x7f.1/ HTTP/1.0\r\n\r\n", 400],
+        ["GET http://127.01.0.1/ HTTP/1.0\r\n\r\n", 400],
+        ["GET http://%3127.0.0.1/ HTTP/1.0\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nHost: localhost:80\r\nHost: localhost:80\r\n\r\n", 400],
+        ["GET http://foo@localhost:80/ HTTP/1.0\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nHost: 4714::abcd:8001\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nHost: abc\xa0\r\n\r\n", 400],
+        ["GET / HTTP/1.0\r\nHost: foo_bar.example.com\r\n\r\n", 200],
+        ["GET http://foo_bar/ HTTP/1.0\r\n\r\n", 200],
+    ])
+    def test_h1_007_01(self, env, intext, status: Optional[int]):
+        with socket.create_connection(('localhost', int(env.http_port))) as sock:
+            # on some OS, the server does not see our connection until there is
+            # something incoming
+            sock.sendall(intext.encode())
+            sock.shutdown(socket.SHUT_WR)
+            buff = sock.recv(1024)
+            msg = buff.decode()
+            if status is None:
+                assert len(msg) == 0, f"unexpected answer: {msg}"
+            else:
+                assert len(msg) > 0, "no answer from server"
+                rlines = msg.splitlines()
+                response = rlines[0]
+                m = re.match(r'^HTTP/1.1 (\d+)\s+(\S+)', response)
+                assert m, f"unrecognized response: {rlines}"
+                if status == 1:
+                    assert int(m.group(1)) >= 200
+                elif status == 90:
+                    assert len(rlines) >= 1, f"{rlines}"
+                elif status > 0:
+                    assert int(m.group(1)) == status, f"{rlines}"
+                else:
+                    assert int(m.group(1)) >= 400, f"{rlines}"
diff --git a/test/modules/http2/env.py b/test/modules/http2/env.py
index 15ef5d3650..d3a8a65dd9 100644
--- a/test/modules/http2/env.py
+++ b/test/modules/http2/env.py
@@ -61,15 +61,15 @@ class H2TestEnv(HttpdTestEnv):
     @property
     def is_unsupported(cls):
         mpm_module = f"mpm_{os.environ['MPM']}" if 'MPM' in os.environ else 'mpm_event'
-        return mpm_module in ['mpm_prefork']
+        return mpm_module == 'mpm_prefork'
 
     def __init__(self, pytestconfig=None):
         super().__init__(pytestconfig=pytestconfig)
         self.add_httpd_conf([
-                             "H2MinWorkers 1",
-                             "H2MaxWorkers 64",
-                             "Protocols h2 http/1.1 h2c",
-                         ])
+            "H2MinWorkers 1",
+            "H2MaxWorkers 64",
+            "Protocols h2 http/1.1 h2c",
+        ])
         self.add_httpd_log_modules(["http2", "proxy_http2", "h2test", "proxy", "proxy_http"])
         self.add_cert_specs([
             CertificateSpec(domains=[
@@ -115,6 +115,12 @@ class H2Conf(HttpdConf):
             f"cgi.{env.http_tld}": [
                 "SSLOptions +StdEnvVars",
                 "AddHandler cgi-script .py",
+                "<Location \"/h2test/echo\">",
+                "    SetHandler h2test-echo",
+                "</Location>",
+                "<Location \"/h2test/delay\">",
+                "    SetHandler h2test-delay",
+                "</Location>",
             ]
         }))
 
diff --git a/test/pyhttpd/conf.py b/test/pyhttpd/conf.py
index 5b5b4ec271..ae34e78b4b 100644
--- a/test/pyhttpd/conf.py
+++ b/test/pyhttpd/conf.py
@@ -157,12 +157,6 @@ class HttpdConf(object):
         self.start_vhost(domains=[domain, f"cgi-alias.{self.env.http_tld}"],
                          port=self.env.https_port, doc_root="htdocs/cgi")
         self.add_proxies("cgi", proxy_self=proxy_self, h2proxy_self=h2proxy_self)
-        self.add("<Location \"/h2test/echo\">")
-        self.add("    SetHandler h2test-echo")
-        self.add("</Location>")
-        self.add("<Location \"/h2test/delay\">")
-        self.add("    SetHandler h2test-delay")
-        self.add("</Location>")
         if domain in self._extras:
             self.add(self._extras[domain])
         self.end_vhost()
diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py
index 89d04dd95e..45f6d2f066 100644
--- a/test/pyhttpd/env.py
+++ b/test/pyhttpd/env.py
@@ -1,3 +1,4 @@
+import importlib
 import inspect
 import logging
 import re
@@ -68,6 +69,7 @@ class HttpdTestSetup:
         self.env = env
         self._source_dirs = [os.path.dirname(inspect.getfile(HttpdTestSetup))]
         self._modules = HttpdTestSetup.MODULES.copy()
+        self._optional_modules = []
 
     def add_source_dir(self, source_dir):
         self._source_dirs.append(source_dir)
@@ -75,6 +77,9 @@ class HttpdTestSetup:
     def add_modules(self, modules: List[str]):
         self._modules.extend(modules)
 
+    def add_optional_modules(self, modules: List[str]):
+        self._optional_modules.extend(modules)
+
     def make(self):
         self._make_dirs()
         self._make_conf()
@@ -141,6 +146,16 @@ class HttpdTestSetup:
                 else:
                     fd.write(f"#built static: LoadModule {m}_module   \"{mod_path}\"\n")
                 loaded.add(m)
+            for m in self._optional_modules:
+                match = re.match(r'^mod_(.+)$', m)
+                if match:
+                    m = match.group(1)
+                if m in loaded:
+                    continue
+                mod_path = os.path.join(self.env.libexec_dir, f"mod_{m}.so")
+                if os.path.isfile(mod_path):
+                    fd.write(f"LoadModule {m}_module   \"{mod_path}\"\n")
+                    loaded.add(m)
         if len(missing_mods) > 0:
             raise Exception(f"Unable to find modules: {missing_mods} "
                             f"DSOs: {self.env.dso_modules}")
@@ -167,10 +182,32 @@ class HttpdTestSetup:
 
 class HttpdTestEnv:
 
+    LIBEXEC_DIR = None
+
+    @classmethod
+    def has_python_package(cls, name: str) -> bool:
+        if name in sys.modules:
+            # already loaded
+            return True
+        elif (spec := importlib.util.find_spec(name)) is not None:
+            module = importlib.util.module_from_spec(spec)
+            sys.modules[name] = module
+            spec.loader.exec_module(module)
+            return True
+        else:
+            return False
+
     @classmethod
     def get_ssl_module(cls):
         return os.environ['SSL'] if 'SSL' in os.environ else 'mod_ssl'
 
+    @classmethod
+    def has_shared_module(cls, name):
+        if cls.LIBEXEC_DIR is None:
+            env = HttpdTestEnv()  # will initialized it
+        path = os.path.join(cls.LIBEXEC_DIR, f"mod_{name}.so")
+        return os.path.isfile(path)
+
     def __init__(self, pytestconfig=None):
         self._our_dir = os.path.dirname(inspect.getfile(Dummy))
         self.config = ConfigParser(interpolation=ExtendedInterpolation())
@@ -180,8 +217,8 @@ class HttpdTestEnv:
         self._apxs = self.config.get('global', 'apxs')
         self._prefix = self.config.get('global', 'prefix')
         self._apachectl = self.config.get('global', 'apachectl')
-        self._libexec_dir = self.get_apxs_var('LIBEXECDIR')
-
+        if HttpdTestEnv.LIBEXEC_DIR is None:
+            HttpdTestEnv.LIBEXEC_DIR = self._libexec_dir = self.get_apxs_var('LIBEXECDIR')
         self._curl = self.config.get('global', 'curl_bin')
         self._nghttp = self.config.get('global', 'nghttp')
         if self._nghttp is None:
@@ -332,7 +369,7 @@ class HttpdTestEnv:
 
     @property
     def libexec_dir(self) -> str:
-        return self._libexec_dir
+        return HttpdTestEnv.LIBEXEC_DIR
 
     @property
     def dso_modules(self) -> List[str]:
@@ -604,32 +641,56 @@ class HttpdTestEnv:
 
     def curl_parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecResult:
         lines = open(headerfile).readlines()
-        exp_stat = True
         if r is None:
             r = ExecResult(args=[], exit_code=0, stdout=b'', stderr=b'')
-        header = {}
+
+        response = None
+        def fin_response(response):
+            if response:
+                r.add_response(response)
+
+        expected = ['status']
         for line in lines:
-            if exp_stat:
+            if re.match(r'^$', line):
+                if 'trailer' in expected:
+                    # end of trailers
+                    fin_response(response)
+                    response = None
+                    expected = ['status']
+                elif 'header' in expected:
+                    # end of header, another status or trailers might follow
+                    expected = ['status', 'trailer']
+                else:
+                    assert False, f"unexpected line: {line}"
+                continue
+            if 'status' in expected:
                 log.debug("reading 1st response line: %s", line)
                 m = re.match(r'^(\S+) (\d+) (.*)$', line)
-                assert m
-                r.add_response({
-                    "protocol": m.group(1),
-                    "status": int(m.group(2)),
-                    "description": m.group(3),
-                    "body": r.outraw
-                })
-                exp_stat = False
-                header = {}
-            elif re.match(r'^$', line):
-                exp_stat = True
-            else:
-                log.debug("reading header line: %s", line)
+                if m:
+                    fin_response(response)
+                    response = {
+                        "protocol": m.group(1),
+                        "status": int(m.group(2)),
+                        "description": m.group(3),
+                        "header": {},
+                        "trailer": {},
+                        "body": r.outraw
+                    }
+                    expected = ['header']
+                    continue
+            if 'trailer' in expected:
+                m = re.match(r'^([^:]+):\s*(.*)$', line)
+                if m:
+                    response['trailer'][m.group(1).lower()] = m.group(2)
+                    continue
+            if 'header' in expected:
                 m = re.match(r'^([^:]+):\s*(.*)$', line)
-                assert m
-                header[m.group(1).lower()] = m.group(2)
-        if r.response:
-            r.response["header"] = header
+                if m:
+                    response['header'][m.group(1).lower()] = m.group(2)
+                    continue
+            assert False, f"unexpected line: {line}"
+
+        fin_response(response)
         return r
 
     def curl_raw(self, urls, timeout=10, options=None, insecure=False,