diff options
author | Stefan Eissing <icing@apache.org> | 2021-10-07 12:43:52 +0000 |
---|---|---|
committer | Stefan Eissing <icing@apache.org> | 2021-10-07 12:43:52 +0000 |
commit | e01aaba8bc72876dc13dbb107abc616e4f14b840 (patch) | |
tree | 9c89095a92c94703f9200398c3a7bd80e8bce820 | |
parent | 8c9b41ffb34b101f9d6d3e838766e71bccf974e0 (diff) | |
download | httpd-e01aaba8bc72876dc13dbb107abc616e4f14b840.tar.gz |
* update of test/modules/http2 from trunk.
git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1893984 13f79535-47bb-0310-9956-ffa450edef68
-rw-r--r-- | test/modules/http2/conftest.py | 26 | ||||
-rw-r--r-- | test/modules/http2/h2_conf.py | 26 | ||||
-rw-r--r-- | test/modules/http2/h2_curl.py | 133 | ||||
-rw-r--r-- | test/modules/http2/h2_env.py | 102 | ||||
-rwxr-xr-x | test/modules/http2/htdocs/test2/006/006.css | 21 | ||||
-rw-r--r-- | test/modules/http2/htdocs/test2/10%abnormal.txt | 0 | ||||
-rw-r--r-- | test/modules/http2/htdocs/test2/x%2f.test | 0 | ||||
-rw-r--r-- | test/modules/http2/mod_h2test/mod_h2test.c | 90 | ||||
-rw-r--r-- | test/modules/http2/test_004_post.py | 51 | ||||
-rw-r--r-- | test/modules/http2/test_105_timeout.py | 54 | ||||
-rw-r--r-- | test/modules/http2/test_202_trailer.py | 16 | ||||
-rw-r--r-- | test/modules/http2/test_203_encoding.py | 105 | ||||
-rw-r--r-- | test/modules/http2/test_600_h2proxy.py | 4 | ||||
-rw-r--r-- | test/modules/http2/test_710_load_post_static.py | 6 | ||||
-rw-r--r-- | test/modules/http2/test_712_buffering.py | 137 |
15 files changed, 544 insertions, 227 deletions
diff --git a/test/modules/http2/conftest.py b/test/modules/http2/conftest.py index 2b8bb057f2..363abae4c2 100644 --- a/test/modules/http2/conftest.py +++ b/test/modules/http2/conftest.py @@ -7,17 +7,22 @@ from h2_certs import CertificateSpec, H2TestCA from h2_env import H2TestEnv -class Dummy: - pass +def pytest_report_header(config, startdir): + env = H2TestEnv(setup_dirs=False) + return f"mod_h2 [apache: {env.get_httpd_version()}, mpm: {env.mpm_type}, {env.prefix}]" -def pytest_report_header(config, startdir): - env = H2TestEnv() - return "mod_h2 [apache: {aversion}({prefix}), mpm: {mpm}]".format( - prefix=env.prefix, - aversion=env.get_httpd_version(), - mpm=env.mpm_type - ) +def pytest_addoption(parser): + parser.addoption("--repeat", action="store", type=int, default=1, + help='Number of times to repeat each test') + parser.addoption("--all", action="store_true") + + +def pytest_generate_tests(metafunc): + if "repeat" in metafunc.fixturenames: + count = int(metafunc.config.getoption("repeat")) + metafunc.fixturenames.append('tmp_ct') + metafunc.parametrize('repeat', range(count)) @pytest.fixture(scope="session") @@ -29,7 +34,6 @@ def env(pytestconfig) -> H2TestEnv: logging.getLogger('').addHandler(console) logging.getLogger('').setLevel(level=level) env = H2TestEnv(pytestconfig=pytestconfig) - env.apache_error_log_clear() cert_specs = [ CertificateSpec(domains=env.domains, key_type='rsa4096'), CertificateSpec(domains=env.domains_noh2, key_type='rsa2048'), @@ -38,6 +42,8 @@ def env(pytestconfig) -> H2TestEnv: store_dir=os.path.join(env.server_dir, 'ca'), key_type="rsa4096") ca.issue_certs(cert_specs) env.set_ca(ca) + env.apache_access_log_clear() + env.apache_error_log_clear() return env diff --git a/test/modules/http2/h2_conf.py b/test/modules/http2/h2_conf.py index 743ccc8c43..4edfaa2dd0 100644 --- a/test/modules/http2/h2_conf.py +++ b/test/modules/http2/h2_conf.py @@ -87,21 +87,29 @@ class HttpdConf(object): self.end_vhost() return self - def add_vhost_test2(self): + def add_vhost_test2(self, extras=None): + domain = f"test2.{self.env.http_tld}" + if extras and 'base' in extras: + self.add(extras['base']) self.start_vhost(self.env.http_port, "test2", aliases=["www2"], doc_root="htdocs/test2", with_ssl=False) self.add(" Protocols http/1.1 h2c") self.end_vhost() self.start_vhost(self.env.https_port, "test2", aliases=["www2"], doc_root="htdocs/test2", with_ssl=True) - self.add(""" + self.add(f""" Protocols http/1.1 h2 <Location /006> Options +Indexes HeaderName /006/header.html - </Location>""") + </Location> + {extras[domain] if extras and domain in extras else ""} + """) self.end_vhost() return self - def add_vhost_cgi(self, proxy_self=False, h2proxy_self=False): + def add_vhost_cgi(self, proxy_self=False, h2proxy_self=False, extras=None): + domain = f"cgi.{self.env.http_tld}" + if extras and 'base' in extras: + self.add(extras['base']) if proxy_self: self.add_proxy_setup() if h2proxy_self: @@ -115,13 +123,21 @@ class HttpdConf(object): <Location \"/.well-known/h2/state\"> SetHandler http2-status </Location>""") - self.add_proxies("cgi", proxy_self, h2proxy_self) + self.add_proxies("cgi", proxy_self=proxy_self, h2proxy_self=h2proxy_self) self.add(" <Location \"/h2test/echo\">") self.add(" SetHandler h2test-echo") self.add(" </Location>") + self.add(" <Location \"/h2test/delay\">") + self.add(" SetHandler h2test-delay") + self.add(" </Location>") + if extras and domain in extras: + self.add(extras[domain]) self.end_vhost() self.start_vhost(self.env.http_port, "cgi", aliases=["cgi-alias"], doc_root="htdocs/cgi", with_ssl=False) self.add(" AddHandler cgi-script .py") + self.add_proxies("cgi", proxy_self=proxy_self, h2proxy_self=h2proxy_self) + if extras and domain in extras: + self.add(extras[domain]) self.end_vhost() self.add(" LogLevel proxy:info") self.add(" LogLevel proxy_http:info") diff --git a/test/modules/http2/h2_curl.py b/test/modules/http2/h2_curl.py new file mode 100644 index 0000000000..fcabe7632a --- /dev/null +++ b/test/modules/http2/h2_curl.py @@ -0,0 +1,133 @@ +import datetime +import re +import subprocess +import sys +import time +from threading import Thread + +from h2_env import H2TestEnv + + +class CurlPiper: + + def __init__(self, env: H2TestEnv, url: str): + self.env = env + self.url = url + self.proc = None + self.args = None + self.headerfile = None + self._stderr = [] + self._stdout = [] + self.stdout_thread = None + self.stderr_thread = None + self._exitcode = -1 + self._r = None + + @property + def exitcode(self): + return self._exitcode + + @property + def response(self): + return self._r.response if self._r else None + + def start(self): + self.args, self.headerfile = self.env.curl_complete_args(self.url, timeout=5, options=[ + "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time"]) + sys.stderr.write("starting: {0}\n".format(self.args)) + self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0) + + def read_output(fh, buffer): + while True: + chunk = fh.read() + if not chunk: + break + buffer.append(chunk.decode()) + + # collect all stdout and stderr until we are done + # use separate threads to not block ourself + self._stderr = [] + self._stdout = [] + if self.proc.stderr: + self.stderr_thread = Thread(target=read_output, args=(self.proc.stderr, self._stderr)) + self.stderr_thread.start() + if self.proc.stdout: + self.stdout_thread = Thread(target=read_output, args=(self.proc.stdout, self._stdout)) + self.stdout_thread.start() + return self.proc + + def send(self, data: str): + self.proc.stdin.write(data.encode()) + self.proc.stdin.flush() + + def close(self) -> ([str], [str]): + self.proc.stdin.close() + self.stdout_thread.join() + self.stderr_thread.join() + self._end() + return self._stdout, self._stderr + + def _end(self): + if self.proc: + # noinspection PyBroadException + try: + if self.proc.stdin: + # noinspection PyBroadException + try: + self.proc.stdin.close() + except Exception: + pass + if self.proc.stdout: + self.proc.stdout.close() + if self.proc.stderr: + self.proc.stderr.close() + except Exception: + self.proc.terminate() + finally: + self.proc.wait() + self.stdout_thread = None + self.stderr_thread = None + self._exitcode = self.proc.returncode + self.proc = None + self._r = self.env.curl_parse_headerfile(self.headerfile) + + def stutter_check(self, chunks: [str], stutter: datetime.timedelta): + if not self.proc: + self.start() + for chunk in chunks: + self.send(chunk) + time.sleep(stutter.total_seconds()) + recv_out, recv_err = self.close() + # assert we got everything back + assert "".join(chunks) == "".join(recv_out) + # now the tricky part: check *when* we got everything back + recv_times = [] + for line in "".join(recv_err).split('\n'): + m = re.match(r'^\s*(\d+:\d+:\d+(\.\d+)?) <= Recv data, (\d+) bytes.*', line) + if m: + recv_times.append(datetime.time.fromisoformat(m.group(1))) + # received as many chunks as we sent + assert len(chunks) == len(recv_times), "received response not in {0} chunks, but {1}".format( + len(chunks), len(recv_times)) + + def microsecs(tdelta): + return ((tdelta.hour * 60 + tdelta.minute) * 60 + tdelta.second) * 1000000 + tdelta.microsecond + + recv_deltas = [] + last_mics = microsecs(recv_times[0]) + for ts in recv_times[1:]: + mics = microsecs(ts) + delta_mics = mics - last_mics + if delta_mics < 0: + delta_mics += datetime.time(23, 59, 59, 999999) + recv_deltas.append(datetime.timedelta(microseconds=delta_mics)) + last_mics = mics + stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.9) # 10% leeway + # TODO: the first two chunks are often close together, it seems + # there still is a little buffering delay going on + for idx, td in enumerate(recv_deltas[1:]): + assert stutter_td < td, \ + f"chunk {idx} arrived too early \n{recv_deltas}\nafter {td}\n{recv_err}" diff --git a/test/modules/http2/h2_env.py b/test/modules/http2/h2_env.py index ba0c870c67..7f879870e7 100644 --- a/test/modules/http2/h2_env.py +++ b/test/modules/http2/h2_env.py @@ -123,9 +123,13 @@ class H2TestSetup: os.chmod(cgi_file, st.st_mode | stat.S_IEXEC) def _make_h2test(self): - subprocess.run([self.env.apxs, '-c', 'mod_h2test.c'], - capture_output=True, check=True, - cwd=os.path.join(self.env.test_dir, 'mod_h2test')) + p = subprocess.run([self.env.apxs, '-c', 'mod_h2test.c'], + capture_output=True, + cwd=os.path.join(self.env.test_dir, 'mod_h2test')) + rv = p.returncode + if rv != 0: + log.error(f"compiling md_h2test failed: {p.stderr}") + raise Exception(f"compiling md_h2test failed: {p.stderr}") def _make_modules_conf(self): modules_conf = os.path.join(self.env.server_dir, 'conf/modules.conf') @@ -143,7 +147,7 @@ class H2TestSetup: class H2TestEnv: - def __init__(self, pytestconfig=None): + def __init__(self, pytestconfig=None, setup_dirs=True): our_dir = os.path.dirname(inspect.getfile(Dummy)) self.config = ConfigParser(interpolation=ExtendedInterpolation()) self.config.read(os.path.join(our_dir, 'config.ini')) @@ -168,6 +172,7 @@ class H2TestEnv: self._server_conf_dir = os.path.join(self._server_dir, "conf") self._server_docs_dir = os.path.join(self._server_dir, "htdocs") self._server_logs_dir = os.path.join(self.server_dir, "logs") + self._server_access_log = os.path.join(self._server_logs_dir, "access_log") self._server_error_log = os.path.join(self._server_logs_dir, "error_log") self._dso_modules = self.config.get('global', 'dso_modules').split(' ') @@ -201,26 +206,31 @@ class H2TestEnv: H2MaxWorkers 64 SSLSessionCache "shmcb:ssl_gcache_data(32000)" """ - py_verbosity = pytestconfig.option.verbose if pytestconfig is not None else 0 - if py_verbosity >= 2: + self._verbosity = pytestconfig.option.verbose if pytestconfig is not None else 0 + if self._verbosity >= 2: self._httpd_base_conf += f""" - LogLevel http2:trace2 proxy_http2:info + LogLevel http2:trace2 proxy_http2:info h2test:trace2 LogLevel core:trace5 mpm_{self.mpm_type}:trace5 """ - if py_verbosity >= 1: - self._httpd_base_conf += "LogLevel http2:debug proxy_http2:debug" + elif self._verbosity >= 1: + self._httpd_base_conf += "LogLevel http2:debug proxy_http2:debug h2test:debug" else: self._httpd_base_conf += "LogLevel http2:info proxy_http2:info" self._verify_certs = False - self._setup = H2TestSetup(env=self) - self._setup.make() + if setup_dirs: + self._setup = H2TestSetup(env=self) + self._setup.make() @property def apxs(self) -> str: return self._apxs @property + def verbosity(self) -> int: + return self._verbosity + + @property def prefix(self) -> str: return self._prefix @@ -397,7 +407,7 @@ class H2TestEnv: req = requests.Request('HEAD', url).prepare() s.send(req, verify=self._verify_certs, timeout=int(timeout.total_seconds())) time.sleep(.2) - except IOError as ex: + except IOError: return True log.debug("Server still responding after %d sec", timeout) return False @@ -422,7 +432,7 @@ class H2TestEnv: return rv def apache_restart(self): - rv = self.apache_stop() + self.apache_stop() rv = self._run_apachectl("start") if rv == 0: timeout = timedelta(seconds=10) @@ -437,6 +447,10 @@ class H2TestEnv: log.debug("waited for a apache.is_dead, rv=%d", rv) return rv + def apache_access_log_clear(self): + if os.path.isfile(self._server_access_log): + os.remove(self._server_access_log) + def apache_error_log_clear(self): if os.path.isfile(self._server_error_log): os.remove(self._server_error_log) @@ -494,41 +508,51 @@ class H2TestEnv: "--cacert", self.ca.cert_file, "-s", "-D", headerfile, "--resolve", ("%s:%s:%s" % (u.hostname, u.port, self._httpd_addr)), - "--connect-timeout", ("%d" % timeout) + "--connect-timeout", ("%d" % timeout), + "--path-as-is" ] if options: args.extend(options) args += urls return args, headerfile + def curl_parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecResult: + lines = open(headerfile).readlines() + exp_stat = True + if r is None: + r = ExecResult(exit_code=0, stdout=b'', stderr=b'') + header = {} + for line in lines: + if exp_stat: + log.debug("reading 1st response line: %s", line) + m = re.match(r'^(\S+) (\d+) (.*)$', line) + assert m + r.add_response({ + "protocol": m.group(1), + "status": int(m.group(2)), + "description": m.group(3), + "body": r.outraw + }) + exp_stat = False + header = {} + elif re.match(r'^$', line): + exp_stat = True + else: + log.debug("reading header line: %s", line) + m = re.match(r'^([^:]+):\s*(.*)$', line) + assert m + header[m.group(1).lower()] = m.group(2) + r.response["header"] = header + return r + def curl_raw(self, urls, timeout, options): - args, headerfile = self.curl_complete_args(urls, timeout, options) + xopt = ['-vvvv'] + if options: + xopt.extend(options) + args, headerfile = self.curl_complete_args(urls, timeout, xopt) r = self.run(args) if r.exit_code == 0: - lines = open(headerfile).readlines() - exp_stat = True - header = {} - for line in lines: - if exp_stat: - log.debug("reading 1st response line: %s", line) - m = re.match(r'^(\S+) (\d+) (.*)$', line) - assert m - r.add_response({ - "protocol": m.group(1), - "status": int(m.group(2)), - "description": m.group(3), - "body": r.outraw - }) - exp_stat = False - header = {} - elif re.match(r'^$', line): - exp_stat = True - else: - log.debug("reading header line: %s", line) - m = re.match(r'^([^:]+):\s*(.*)$', line) - assert m - header[m.group(1).lower()] = m.group(2) - r.response["header"] = header + self.curl_parse_headerfile(headerfile, r=r) if r.json: r.response["json"] = r.json return r diff --git a/test/modules/http2/htdocs/test2/006/006.css b/test/modules/http2/htdocs/test2/006/006.css new file mode 100755 index 0000000000..de6aa5fd18 --- /dev/null +++ b/test/modules/http2/htdocs/test2/006/006.css @@ -0,0 +1,21 @@ +@CHARSET "ISO-8859-1";
+body{
+ background:HoneyDew;
+}
+p{
+color:#0000FF;
+text-align:left;
+}
+
+h1{
+color:#FF0000;
+text-align:center;
+}
+
+.listTitle{
+ font-size:large;
+}
+
+.listElements{
+ color:#3366FF
+}
\ No newline at end of file diff --git a/test/modules/http2/htdocs/test2/10%abnormal.txt b/test/modules/http2/htdocs/test2/10%abnormal.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/modules/http2/htdocs/test2/10%abnormal.txt diff --git a/test/modules/http2/htdocs/test2/x%2f.test b/test/modules/http2/htdocs/test2/x%2f.test new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/modules/http2/htdocs/test2/x%2f.test diff --git a/test/modules/http2/mod_h2test/mod_h2test.c b/test/modules/http2/mod_h2test/mod_h2test.c index 7cd66d1286..b65f1df6f7 100644 --- a/test/modules/http2/mod_h2test/mod_h2test.c +++ b/test/modules/http2/mod_h2test/mod_h2test.c @@ -17,6 +17,7 @@ #include <apr_optional.h> #include <apr_optional_hooks.h> #include <apr_strings.h> +#include <apr_cstr.h> #include <apr_time.h> #include <apr_want.h> @@ -143,6 +144,92 @@ cleanup: return DECLINED; } +static int h2test_delay_handler(request_rec *r) +{ + conn_rec *c = r->connection; + apr_bucket_brigade *bb; + apr_bucket *b; + apr_status_t rv; + char buffer[8192]; + int i, chunks = 3; + long l; + apr_time_t delay = 0; + + if (strcmp(r->handler, "h2test-delay")) { + return DECLINED; + } + if (r->method_number != M_GET && r->method_number != M_POST) { + return DECLINED; + } + + if (r->args) { + rv = apr_cstr_atoi(&i, r->args); + if (APR_SUCCESS == rv) { + delay = apr_time_from_sec(i); + } + } + + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "delay_handler: processing request, %ds delay", + (int)apr_time_sec(delay)); + r->status = 200; + r->clength = -1; + r->chunked = 1; + apr_table_unset(r->headers_out, "Content-Length"); + /* Discourage content-encodings */ + apr_table_unset(r->headers_out, "Content-Encoding"); + apr_table_setn(r->subprocess_env, "no-brotli", "1"); + apr_table_setn(r->subprocess_env, "no-gzip", "1"); + + ap_set_content_type(r, "application/octet-stream"); + + bb = apr_brigade_create(r->pool, c->bucket_alloc); + /* copy any request body into the response */ + if ((rv = ap_setup_client_block(r, REQUEST_CHUNKED_DECHUNK))) goto cleanup; + if (ap_should_client_block(r)) { + do { + l = ap_get_client_block(r, &buffer[0], sizeof(buffer)); + if (l > 0) { + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, + "delay_handler: reading %ld bytes from request body", l); + } + } while (l > 0); + if (l < 0) { + return AP_FILTER_ERROR; + } + } + + memset(buffer, 0, sizeof(buffer)); + l = sizeof(buffer); + for (i = 0; i < chunks; ++i) { + rv = apr_brigade_write(bb, NULL, NULL, buffer, l); + if (APR_SUCCESS != rv) goto cleanup; + rv = ap_pass_brigade(r->output_filters, bb); + if (APR_SUCCESS != rv) goto cleanup; + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, + "delay_handler: passed %ld bytes as response body", l); + if (delay) { + apr_sleep(delay); + } + } + /* we are done */ + b = apr_bucket_eos_create(c->bucket_alloc); + APR_BRIGADE_INSERT_TAIL(bb, b); + rv = ap_pass_brigade(r->output_filters, bb); + apr_brigade_cleanup(bb); + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, rv, r, "delay_handler: response passed"); + +cleanup: + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, rv, r, + "delay_handler: request cleanup, r->status=%d, aborte=%d", + r->status, c->aborted); + if (rv == APR_SUCCESS + || r->status != HTTP_OK + || c->aborted) { + return OK; + } + return AP_FILTER_ERROR; +} + /* Install this module into the apache2 infrastructure. */ static void h2test_hooks(apr_pool_t *pool) @@ -159,7 +246,8 @@ static void h2test_hooks(apr_pool_t *pool) */ ap_hook_child_init(h2test_child_init, NULL, NULL, APR_HOOK_MIDDLE); - /* test h2 echo handler */ + /* test h2 handlers */ ap_hook_handler(h2test_echo_handler, NULL, NULL, APR_HOOK_MIDDLE); + ap_hook_handler(h2test_delay_handler, NULL, NULL, APR_HOOK_MIDDLE); } diff --git a/test/modules/http2/test_004_post.py b/test/modules/http2/test_004_post.py index 5f85494a45..16d1c7679c 100644 --- a/test/modules/http2/test_004_post.py +++ b/test/modules/http2/test_004_post.py @@ -20,7 +20,7 @@ class TestStore: url = env.mkurl("https", "cgi", "/upload.py") fpath = os.path.join(env.gen_dir, fname) r = env.curl_upload(url, fpath, options=options) - assert r.exit_code == 0 + assert r.exit_code == 0, r.stderr assert r.response["status"] >= 200 and r.response["status"] < 300 r2 = env.curl_get(r.response["header"]["location"]) @@ -105,17 +105,17 @@ class TestStore: src = file.read() assert src == r.response["body"] - def test_004_21(self, env): - self.nghttp_post_and_verify(env, "data-1k", []) - self.nghttp_post_and_verify(env, "data-10k", []) - self.nghttp_post_and_verify(env, "data-100k", []) - self.nghttp_post_and_verify(env, "data-1m", []) + @pytest.mark.parametrize("name", [ + "data-1k", "data-10k", "data-100k", "data-1m" + ]) + def test_004_21(self, env, name): + self.nghttp_post_and_verify(env, name, []) - def test_004_22(self, env): - self.nghttp_post_and_verify(env, "data-1k", ["--no-content-length"]) - self.nghttp_post_and_verify(env, "data-10k", ["--no-content-length"]) - self.nghttp_post_and_verify(env, "data-100k", ["--no-content-length"]) - self.nghttp_post_and_verify(env, "data-1m", ["--no-content-length"]) + @pytest.mark.parametrize("name", [ + "data-1k", "data-10k", "data-100k", "data-1m" + ]) + def test_004_22(self, env, name, repeat): + self.nghttp_post_and_verify(env, name, ["--no-content-length"]) # upload and GET again using nghttp, compare to original content def nghttp_upload_and_verify(self, env, fname, options=None): @@ -134,22 +134,23 @@ class TestStore: src = file.read() assert src == r2.response["body"] - def test_004_23(self, env): - self.nghttp_upload_and_verify(env, "data-1k", []) - self.nghttp_upload_and_verify(env, "data-10k", []) - self.nghttp_upload_and_verify(env, "data-100k", []) - self.nghttp_upload_and_verify(env, "data-1m", []) + @pytest.mark.parametrize("name", [ + "data-1k", "data-10k", "data-100k", "data-1m" + ]) + def test_004_23(self, env, name, repeat): + self.nghttp_upload_and_verify(env, name, []) - def test_004_24(self, env): - self.nghttp_upload_and_verify(env, "data-1k", ["--expect-continue"]) - self.nghttp_upload_and_verify(env, "data-100k", ["--expect-continue"]) + @pytest.mark.parametrize("name", [ + "data-1k", "data-10k", "data-100k", "data-1m" + ]) + def test_004_24(self, env, name, repeat): + self.nghttp_upload_and_verify(env, name, ["--expect-continue"]) - @pytest.mark.skipif(True, reason="python3 regresses in chunked inputs to cgi") - def test_004_25(self, env): - self.nghttp_upload_and_verify(env, "data-1k", ["--no-content-length"]) - self.nghttp_upload_and_verify(env, "data-10k", ["--no-content-length"]) - self.nghttp_upload_and_verify(env, "data-100k", ["--no-content-length"]) - self.nghttp_upload_and_verify(env, "data-1m", ["--no-content-length"]) + @pytest.mark.parametrize("name", [ + "data-1k", "data-10k", "data-100k", "data-1m" + ]) + def test_004_25(self, env, name, repeat): + self.nghttp_upload_and_verify(env, name, ["--no-content-length"]) def test_004_30(self, env): # issue: #203 diff --git a/test/modules/http2/test_105_timeout.py b/test/modules/http2/test_105_timeout.py index 85989eb0b2..88a609376b 100644 --- a/test/modules/http2/test_105_timeout.py +++ b/test/modules/http2/test_105_timeout.py @@ -1,7 +1,10 @@ import socket +import time + import pytest from h2_conf import HttpdConf +from h2_curl import CurlPiper class TestStore: @@ -94,3 +97,54 @@ class TestStore: "-F", ("wait1=%f" % 1.5), ]) assert 200 == r.response["status"] + + def test_105_10(self, env): + # just a check without delays if all is fine + conf = HttpdConf(env) + conf.add_vhost_cgi() + conf.install() + assert env.apache_restart() == 0 + url = env.mkurl("https", "cgi", "/h2test/delay") + piper = CurlPiper(env=env, url=url) + piper.start() + stdout, stderr = piper.close() + assert piper.exitcode == 0 + assert len("".join(stdout)) == 3 * 8192 + + @pytest.mark.skipif(True, reason="new feature in upcoming http2") + def test_105_11(self, env): + # short connection timeout, longer stream delay + # receiving the first response chunk, then timeout + conf = HttpdConf(env) + conf.add_vhost_cgi() + conf.add("Timeout 1") + conf.install() + assert env.apache_restart() == 0 + url = env.mkurl("https", "cgi", "/h2test/delay?5") + piper = CurlPiper(env=env, url=url) + piper.start() + stdout, stderr = piper.close() + assert len("".join(stdout)) == 8192 + + @pytest.mark.skipif(True, reason="new feature in upcoming http2") + def test_105_12(self, env): + # long connection timeout, short stream timeout + # sending a slow POST + conf = HttpdConf(env) + conf.add_vhost_cgi() + conf.add("Timeout 10") + conf.add("H2StreamTimeout 1") + conf.install() + assert env.apache_restart() == 0 + url = env.mkurl("https", "cgi", "/h2test/delay?5") + piper = CurlPiper(env=env, url=url) + piper.start() + for _ in range(3): + time.sleep(2) + try: + piper.send("0123456789\n") + except BrokenPipeError: + break + piper.close() + assert piper.response + assert piper.response['status'] == 408 diff --git a/test/modules/http2/test_202_trailer.py b/test/modules/http2/test_202_trailer.py index 04fa5419c0..f43aa85080 100644 --- a/test/modules/http2/test_202_trailer.py +++ b/test/modules/http2/test_202_trailer.py @@ -63,19 +63,3 @@ class TestStore: assert 300 > r.response["status"] assert b"X: 4a\n" == r.response["body"] - # The h2 status handler echoes a trailer if it sees a trailer - def test_202_05(self, env): - url = env.mkurl("https", "cgi", "/.well-known/h2/state") - fpath = os.path.join(env.gen_dir, "data-1k") - r = env.nghttp().upload(url, fpath, options=["--trailer", "test: 2"]) - assert 200 == r.response["status"] - assert "1" == r.response["trailer"]["h2-trailers-in"] - - # Check that we can send and receive trailers throuh mod_proxy_http2 - def test_202_06(self, env): - url = env.mkurl("https", "cgi", "/h2proxy/.well-known/h2/state") - fpath = os.path.join(env.gen_dir, "data-1k") - r = env.nghttp().upload(url, fpath, options=["--trailer", "test: 2"]) - assert 200 == r.response["status"] - assert 'trailer' in r.response - assert "1" == r.response['trailer']["h2-trailers-in"] diff --git a/test/modules/http2/test_203_encoding.py b/test/modules/http2/test_203_encoding.py new file mode 100644 index 0000000000..60d96e0bad --- /dev/null +++ b/test/modules/http2/test_203_encoding.py @@ -0,0 +1,105 @@ +import time + +import pytest + +from h2_conf import HttpdConf + + +class TestEncoding: + + EXP_AH10244_ERRS = 0 + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env): + extras = { + 'base': f""" + <Directory "{env.gen_dir}"> + AllowOverride None + Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch + Require all granted + </Directory> + """, + } + conf = HttpdConf(env) + conf.add_vhost_test1(extras=extras) + conf.add_vhost_test2(extras={ + f"test2.{env.http_tld}": "AllowEncodedSlashes on", + }) + conf.add_vhost_cgi(extras={ + f"cgi.{env.http_tld}": f"ScriptAlias /cgi-bin/ {env.gen_dir}", + }) + conf.install() + assert env.apache_restart() == 0 + yield + errors, warnings = env.apache_errors_and_warnings() + assert (len(errors), len(warnings)) == (TestEncoding.EXP_AH10244_ERRS, 0),\ + f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\ + "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) + env.apache_error_log_clear() + + # check handling of url encodings that are accepted + @pytest.mark.parametrize("path", [ + "/006/006.css", + "/%30%30%36/%30%30%36.css", + "/nothing/../006/006.css", + "/nothing/./../006/006.css", + "/nothing/%2e%2e/006/006.css", + "/nothing/%2e/%2e%2e/006/006.css", + "/nothing/%2e/%2e%2e/006/006%2ecss", + ]) + def test_203_01(self, env, path): + url = env.mkurl("https", "test1", path) + r = env.curl_get(url) + assert r.response["status"] == 200 + + # check handling of / normalization + @pytest.mark.parametrize("path", [ + "/006//006.css", + "/006//////////006.css", + "/006////.//////006.css", + "/006////%2e//////006.css", + "/006////%2e//////006%2ecss", + "/006/../006/006.css", + "/006/%2e%2e/006/006.css", + ]) + def test_203_03(self, env, path): + url = env.mkurl("https", "test1", path) + r = env.curl_get(url) + assert r.response["status"] == 200 + + # check path traversals + @pytest.mark.parametrize(["path", "status"], [ + ["/../echo.py", 400], + ["/nothing/../../echo.py", 400], + ["/cgi-bin/../../echo.py", 400], + ["/nothing/%2e%2e/%2e%2e/echo.py", 400], + ["/cgi-bin/%2e%2e/%2e%2e/echo.py", 400], + ["/nothing/%%32%65%%32%65/echo.py", 400], + ["/cgi-bin/%%32%65%%32%65/echo.py", 400], + ["/nothing/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400], + ["/cgi-bin/%%32%65%%32%65/%%32%65%%32%65/h2_env.py", 400], + ["/nothing/%25%32%65%25%32%65/echo.py", 404], + ["/cgi-bin/%25%32%65%25%32%65/echo.py", 404], + ["/nothing/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404], + ["/cgi-bin/%25%32%65%25%32%65/%25%32%65%25%32%65/h2_env.py", 404], + ]) + def test_203_04(self, env, path, status): + url = env.mkurl("https", "cgi", path) + r = env.curl_get(url) + assert r.response["status"] == status + if status == 400: + TestEncoding.EXP_AH10244_ERRS += 1 + # the log will have a core:err about invalid URI path + + # check handling of %2f url encodings that are not decoded by default + @pytest.mark.parametrize(["host", "path", "status"], [ + ["test1", "/006%2f006.css", 404], + ["test2", "/006%2f006.css", 200], + ["test2", "/x%252f.test", 200], + ["test2", "/10%25abnormal.txt", 200], + ]) + def test_203_20(self, env, host, path, status): + url = env.mkurl("https", host, path) + r = env.curl_get(url) + assert r.response["status"] == status + diff --git a/test/modules/http2/test_600_h2proxy.py b/test/modules/http2/test_600_h2proxy.py index 97b832c2bd..be56d04e31 100644 --- a/test/modules/http2/test_600_h2proxy.py +++ b/test/modules/http2/test_600_h2proxy.py @@ -10,8 +10,8 @@ class TestStore: env.setup_data_1k_1m() conf = HttpdConf(env) conf.add_vhost_cgi(h2proxy_self=True) - conf.add("LogLevel proxy_http2:trace2") - conf.add("LogLevel proxy:trace2") + if env.verbosity > 1: + conf.add("LogLevel proxy:trace2 proxy_http2:trace2") conf.install() assert env.apache_restart() == 0 diff --git a/test/modules/http2/test_710_load_post_static.py b/test/modules/http2/test_710_load_post_static.py index 0dfe6b65f1..2b85b0f5eb 100644 --- a/test/modules/http2/test_710_load_post_static.py +++ b/test/modules/http2/test_710_load_post_static.py @@ -31,7 +31,7 @@ class TestStore: m = 1 conn = 1 fname = "data-10k" - args = [env.h2load, "-n", "%d" % n, "-c", "%d" % conn, "-m", "%d" % m, + args = [env.h2load, "-n", f"{n}", "-c", f"{conn}", "-m", f"{m}", f"--base-uri={env.https_base_url}", "-d", os.path.join(env.gen_dir, fname), url] r = env.run(args) @@ -43,7 +43,7 @@ class TestStore: m = 100 conn = 1 fname = "data-1k" - args = [env.h2load, "-n", "%d" % n, "-c", "%d" % conn, "-m", "%d" % m, + args = [env.h2load, "-n", f"{n}", "-c", f"{conn}", "-m", f"{m}", f"--base-uri={env.https_base_url}", "-d", os.path.join(env.gen_dir, fname), url] r = env.run(args) @@ -55,7 +55,7 @@ class TestStore: m = 50 conn = 1 fname = "data-100k" - args = [env.h2load, "-n", "%d" % n, "-c", "%d" % conn, "-m", "%d" % m, + args = [env.h2load, "-n", f"{n}", "-c", f"{conn}", "-m", f"{m}", f"--base-uri={env.https_base_url}", "-d", os.path.join(env.gen_dir, fname), url] r = env.run(args) diff --git a/test/modules/http2/test_712_buffering.py b/test/modules/http2/test_712_buffering.py index 70b73762c2..ebf43239b5 100644 --- a/test/modules/http2/test_712_buffering.py +++ b/test/modules/http2/test_712_buffering.py @@ -1,134 +1,17 @@ -import datetime -import re -import sys -import time -import subprocess - from datetime import timedelta -from threading import Thread import pytest from h2_conf import HttpdConf +from h2_curl import CurlPiper -class CurlPiper: - - def __init__(self, url: str): - self.url = url - self.proc = None - self.args = None - self.headerfile = None - self._stderr = [] - self._stdout = [] - self.stdout_thread = None - self.stderr_thread = None - - def start(self, env): - self.args, self.headerfile = env.curl_complete_args(self.url, timeout=5, options=[ - "-T", "-", "-X", "POST", "--trace-ascii", "%", "--trace-time"]) - sys.stderr.write("starting: {0}\n".format(self.args)) - self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=0) - - def read_output(fh, buffer): - while True: - chunk = fh.read() - if not chunk: - break - buffer.append(chunk.decode()) - - # collect all stdout and stderr until we are done - # use separate threads to not block ourself - self._stderr = [] - self._stdout = [] - if self.proc.stderr: - self.stderr_thread = Thread(target=read_output, args=(self.proc.stderr, self._stderr)) - self.stderr_thread.start() - if self.proc.stdout: - self.stdout_thread = Thread(target=read_output, args=(self.proc.stdout, self._stdout)) - self.stdout_thread.start() - return self.proc - - def send(self, data: str): - self.proc.stdin.write(data.encode()) - self.proc.stdin.flush() - - def close(self) -> ([str], [str]): - self.proc.stdin.close() - self.stdout_thread.join() - self.stderr_thread.join() - self._end() - return self._stdout, self._stderr - - def _end(self): - if self.proc: - # noinspection PyBroadException - try: - if self.proc.stdin: - # noinspection PyBroadException - try: - self.proc.stdin.close() - except Exception: - pass - if self.proc.stdout: - self.proc.stdout.close() - if self.proc.stderr: - self.proc.stderr.close() - except Exception: - self.proc.terminate() - finally: - self.stdout_thread = None - self.stderr_thread = None - self.proc = None - - def stutter_check(self, env, chunks: [str], stutter: datetime.timedelta): - if not self.proc: - self.start(env) - for chunk in chunks: - self.send(chunk) - time.sleep(stutter.total_seconds()) - recv_out, recv_err = self.close() - # assert we got everything back - assert "".join(chunks) == "".join(recv_out) - # now the tricky part: check *when* we got everything back - recv_times = [] - for line in "".join(recv_err).split('\n'): - m = re.match(r'^\s*(\d+:\d+:\d+(\.\d+)?) <= Recv data, (\d+) bytes.*', line) - if m: - recv_times.append(datetime.time.fromisoformat(m.group(1))) - # received as many chunks as we sent - assert len(chunks) == len(recv_times), "received response not in {0} chunks, but {1}".format( - len(chunks), len(recv_times)) - - def microsecs(tdelta): - return ((tdelta.hour * 60 + tdelta.minute) * 60 + tdelta.second) * 1000000 + tdelta.microsecond - - recv_deltas = [] - last_mics = microsecs(recv_times[0]) - for ts in recv_times[1:]: - mics = microsecs(ts) - delta_mics = mics - last_mics - if delta_mics < 0: - delta_mics += datetime.time(23, 59, 59, 999999) - recv_deltas.append(datetime.timedelta(microseconds=delta_mics)) - last_mics = mics - stutter_td = datetime.timedelta(seconds=stutter.total_seconds() * 0.9) # 10% leeway - # TODO: the first two chunks are often close together, it seems - # there still is a little buffering delay going on - for idx, td in enumerate(recv_deltas[1:]): - assert stutter_td < td, \ - f"chunk {idx} arrived too early \n{recv_deltas}\nafter {td}\n{recv_err}" - - -class TestStore: +class TestBuffering: @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): env.setup_data_1k_1m() - conf = HttpdConf(env).add("H2OutputBuffering off") + conf = HttpdConf(env) conf.add_vhost_cgi(h2proxy_self=True).install() assert env.apache_restart() == 0 @@ -151,9 +34,10 @@ class TestStore: base_chunk = "0123456789" chunks = ["chunk-{0:03d}-{1}\n".format(i, base_chunk) for i in range(5)] stutter = timedelta(seconds=0.2) # this is short, but works on my machine (tm) - piper = CurlPiper(url=url) - piper.stutter_check(env, chunks, stutter) + piper = CurlPiper(env=env, url=url) + piper.stutter_check(chunks, stutter) + @pytest.mark.skipif(True, reason="new feature in upcoming http2") def test_712_02(self, env): # same as 712_01 but via mod_proxy_http2 # @@ -161,9 +45,10 @@ class TestStore: base_chunk = "0123456789" chunks = ["chunk-{0:03d}-{1}\n".format(i, base_chunk) for i in range(3)] stutter = timedelta(seconds=0.4) # need a bit more delay since we have the extra connection - piper = CurlPiper(url=url) - piper.stutter_check(env, chunks, stutter) + piper = CurlPiper(env=env, url=url) + piper.stutter_check(chunks, stutter) + @pytest.mark.skipif(True, reason="new feature in upcoming http2") def test_712_03(self, env): # same as 712_02 but with smaller chunks # @@ -171,5 +56,5 @@ class TestStore: base_chunk = "0" chunks = ["ck{0}-{1}\n".format(i, base_chunk) for i in range(3)] stutter = timedelta(seconds=0.4) # need a bit more delay since we have the extra connection - piper = CurlPiper(url=url) - piper.stutter_check(env, chunks, stutter) + piper = CurlPiper(env=env, url=url) + piper.stutter_check(chunks, stutter) |