tests/http: add pytest to GHA and improve tests

- added to: ngtcp2-quictls, ngtcp2-gnutls and the linux varians
  quiche, bearssl, libressl, mbedtls, openssl3, rustls
- added disabled in ngtcp2-wolfssl due to weird SSL_connect() errors
  not reproducable locally

Improvements on pytest:

-  handling of systems with nghttpx in $PATH
   - configure will seach $PATH got nghttpx used in pytest
   - pytest fixes for managing nghttpx without h3 support
   - ngtcp2-wolfssl: use a fully enabled wolfssl build

- lower parallel count for http/1.1 tests, since we do not
   want to test excessive connections.
- check built curl for HTTPS-proxy support in proxy tests
- bearssl does not like one of our critical cert extensions, making
  it non-critical now
- bearssl is too slow for test_12, skipping
- making sure we do h3 tests only when curl and server support is there

Closes #10699

5 files changed, 82 insertions, 142 deletions

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 13f9ae749..bfef08e99 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -51,12 +51,12 @@ jobs:
         build:
         - name: quiche
           install_packages: zlib1g-dev
-          install_steps: quiche
+          install_steps: quiche pytest
           configure: LDFLAGS="-Wl,-rpath,/home/runner/work/curl/curl/quiche/target/release" --with-openssl=/home/runner/work/curl/curl/quiche/quiche/deps/boringssl/src --enable-debug --with-quiche=/home/runner/work/curl/curl/quiche/target/release
 
         - name: bearssl
           install_packages: zlib1g-dev
-          install_steps: bearssl
+          install_steps: bearssl pytest
           configure: LDFLAGS="-Wl,-rpath,$HOME/bear/lib" --with-bearssl=$HOME/bear --enable-debug
 
         - name: bearssl-clang
@@ -66,7 +66,7 @@ jobs:
 
         - name: libressl
           install_packages: zlib1g-dev
-          install_steps: libressl
+          install_steps: libressl pytest
           configure: LDFLAGS="-Wl,-rpath,$HOME/libressl/lib" --with-openssl=$HOME/libressl --enable-debug
 
         - name: libressl-clang
@@ -76,7 +76,7 @@ jobs:
 
         - name: mbedtls
           install_packages: libnghttp2-dev
-          install_steps: mbedtls
+          install_steps: mbedtls pytest
           configure: LDFLAGS="-Wl,-rpath,$HOME/mbed/lib" --with-mbedtls=$HOME/mbed --enable-debug
 
         - name: mbedtls-clang
@@ -91,7 +91,7 @@ jobs:
 
         - name: openssl3
           install_packages: zlib1g-dev
-          install_steps: gcc-11 openssl3
+          install_steps: gcc-11 openssl3 pytest
           configure: LDFLAGS="-Wl,-rpath,$HOME/openssl3/lib64" --with-openssl=$HOME/openssl3 --enable-debug --enable-websockets
 
         - name: openssl3-O3
@@ -114,7 +114,7 @@ jobs:
           configure: LDFLAGS="-Wl,-rpath,$HOME/hyper/target/debug" --with-openssl --with-hyper=$HOME/hyper --enable-debug --enable-websockets
 
         - name: rustls
-          install_steps: rust rustls
+          install_steps: rust rustls pytest
           configure: --with-rustls=$HOME/rustls --enable-debug
 
         - name: Intel compiler - without SSL
@@ -265,6 +265,18 @@ jobs:
         printenv >> $GITHUB_ENV
       name: 'install Intel compilers'
 
+    - if: ${{ contains(matrix.build.install_steps, 'pytest') }}
+      run: |
+        sudo apt-get install apache2 apache2-dev libnghttp2-dev
+        sudo python3 -m pip install impacket pytest cryptography multipart
+        git clone --depth=1 -b master https://github.com/icing/mod_h2
+        cd mod_h2
+        autoreconf -fi
+        ./configure PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig"
+        make
+        sudo make install
+      name: 'install pytest and apach2-dev mod-h2'
+
     - run: autoreconf -fi
       name: 'autoreconf'
 
@@ -284,3 +296,12 @@ jobs:
       name: 'run tests'
       env:
         TFLAGS: "${{ matrix.build.tflags }}"
+
+    - if: ${{ contains(matrix.build.install_steps, 'pytest') }}
+      # run for `tests` directory, so pytest does not pick up any other
+      # packages we might have built here
+      run:
+        pytest tests
+      name: 'run pytest'
+      env:
+        TFLAGS: "${{ matrix.build.tflags }}"
diff --git a/.github/workflows/ngtcp2-gnutls.yml b/.github/workflows/ngtcp2-gnutls.yml
index 17f247112..c80fd151b 100644
--- a/.github/workflows/ngtcp2-gnutls.yml
+++ b/.github/workflows/ngtcp2-gnutls.yml
@@ -70,8 +70,9 @@ jobs:
     - run: |
         sudo apt-get update
         sudo apt-get install libtool autoconf automake pkg-config stunnel4 ${{ matrix.build.install }}
-        sudo python3 -m pip install impacket
-      name: 'install prereqs and impacket'
+        sudo apt-get install apache2 apache2-dev
+        sudo python3 -m pip install impacket pytest cryptography multipart
+      name: 'install prereqs and impacket, pytest, crypto'
 
     - run: |
         git clone --depth=1 -b openssl-3.0.8+quic https://github.com/quictls/openssl
@@ -120,6 +121,15 @@ jobs:
         make install
       name: 'install nghttp2'
 
+    - run: |
+        git clone --depth=1 -b master https://github.com/icing/mod_h2
+        cd mod_h2
+        autoreconf -fi
+        ./configure PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig"
+        make
+        sudo make install
+      name: 'install mod_h2'
+
     - uses: actions/checkout@v3
 
     - run: autoreconf -fi
@@ -141,3 +151,8 @@ jobs:
       name: 'run tests'
       env:
         TFLAGS: "${{ matrix.build.tflags }}"
+
+    - run: pytest -v
+      name: 'run pytest'
+      env:
+        TFLAGS: "${{ matrix.build.tflags }}"
diff --git a/.github/workflows/ngtcp2-quictls.yml b/.github/workflows/ngtcp2-quictls.yml
index 913a100b7..60b406fc6 100644
--- a/.github/workflows/ngtcp2-quictls.yml
+++ b/.github/workflows/ngtcp2-quictls.yml
@@ -61,8 +61,9 @@ jobs:
     - run: |
         sudo apt-get update
         sudo apt-get install libtool autoconf automake pkg-config stunnel4 ${{ matrix.build.install }}
-        sudo python3 -m pip install impacket
-      name: 'install prereqs and impacket'
+        sudo apt-get install apache2 apache2-dev
+        sudo python3 -m pip install impacket pytest cryptography multipart
+      name: 'install prereqs and impacket, pytest, crypto'
 
     - run: |
         git clone --depth=1 -b openssl-3.0.8+quic https://github.com/quictls/openssl
@@ -95,6 +96,15 @@ jobs:
         make install
       name: 'install nghttp2'
 
+    - run: |
+        git clone --depth=1 -b master https://github.com/icing/mod_h2
+        cd mod_h2
+        autoreconf -fi
+        ./configure PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig"
+        make
+        sudo make install
+      name: 'install mod_h2'
+
     - uses: actions/checkout@v3
 
     - run: autoreconf -fi
@@ -116,3 +126,9 @@ jobs:
       name: 'run tests'
       env:
         TFLAGS: "${{ matrix.build.tflags }}"
+
+    - run: pytest -v
+      name: 'run pytest'
+      env:
+        TFLAGS: "${{ matrix.build.tflags }}"
+
diff --git a/.github/workflows/ngtcp2-wolfssl.yml b/.github/workflows/ngtcp2-wolfssl.yml
index dab8ada02..3cfcdd208 100644
--- a/.github/workflows/ngtcp2-wolfssl.yml
+++ b/.github/workflows/ngtcp2-wolfssl.yml
@@ -59,15 +59,15 @@ jobs:
           ngtcp2-configure: >-
             --prefix=$HOME/all PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig" --enable-lib-only
           wolfssl-configure: >-
-            --enable-quic --enable-session-ticket --enable-earlydata --enable-psk
-            --enable-harden --enable-altcertchains
+            --enable-all --enable-quic
 
     steps:
     - run: |
         sudo apt-get update
         sudo apt-get install libtool autoconf automake pkg-config stunnel4 ${{ matrix.build.install }}
-        sudo python3 -m pip install impacket
-      name: 'install prereqs and impacket'
+        sudo apt-get install apache2 apache2-dev
+        sudo python3 -m pip install impacket pytest cryptography multipart
+      name: 'install prereqs and impacket, pytest, crypto'
 
     - run: |
         git clone https://github.com/wolfSSL/wolfssl.git
@@ -108,6 +108,15 @@ jobs:
         make install
       name: 'install nghttp2'
 
+    - run: |
+        git clone --depth=1 -b master https://github.com/icing/mod_h2
+        cd mod_h2
+        autoreconf -fi
+        ./configure PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig"
+        make
+        sudo make install
+      name: 'install mod_h2'
+
     - uses: actions/checkout@v3
 
     - run: autoreconf -fi
@@ -129,3 +138,10 @@ jobs:
       name: 'run tests'
       env:
         TFLAGS: "${{ matrix.build.tflags }}"
+
+    # Disabled for now, we see spurious SSL_connect() errors when talking
+    # http/1.1 to apache and this, so far, is not reproducable in local testing
+    #- run: pytest -v
+    #  name: 'run pytest'
+    #  env:
+    #    TFLAGS: "${{ matrix.build.tflags }}"
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
deleted file mode 100644
index 28bee2080..000000000
--- a/.github/workflows/pytest.yml
+++ /dev/null
@@ -1,128 +0,0 @@
-# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
-#
-# SPDX-License-Identifier: curl
-
-name: pytest
-
-on:
-  push:
-    branches:
-    - master
-    - '*/ci'
-    paths-ignore:
-    - '**/*.md'
-    - '.azure-pipelines.yml'
-    - '.circleci/**'
-    - '.cirrus.yml'
-    - 'appveyor.yml'
-    - 'packages/**'
-    - 'plan9/**'
-    - 'projects/**'
-    - 'winbuild/**'
-  pull_request:
-    branches:
-    - master
-    paths-ignore:
-    - '**/*.md'
-    - '.azure-pipelines.yml'
-    - '.circleci/**'
-    - '.cirrus.yml'
-    - 'appveyor.yml'
-    - 'packages/**'
-    - 'plan9/**'
-    - 'projects/**'
-    - 'winbuild/**'
-
-concurrency:
-  # Hardcoded workflow filename as workflow name above is just Linux again
-  group: pytest-openssl-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: true
-
-jobs:
-  autotools:
-    name: ${{ matrix.build.name }}
-    runs-on: 'ubuntu-latest'
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        build:
-        - name: quictls
-          install: >-
-            libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libev-dev libc-ares-dev
-          configure: >-
-            PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig" LDFLAGS="-Wl,-rpath,$HOME/all/lib"
-            --with-ngtcp2=$HOME/all --enable-warnings --enable-werror --enable-debug
-            --with-test-nghttpx="$HOME/all/bin/nghttpx"
-          ngtcp2-configure: >-
-            --prefix=$HOME/all PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig" --enable-lib-only
-
-    steps:
-    - run: |
-        sudo apt-get update
-        sudo apt-get install libtool autoconf automake pkg-config stunnel4 ${{ matrix.build.install }}
-        sudo apt-get install apache2 apache2-dev
-        sudo python3 -m pip install impacket pytest cryptography
-      name: 'install prereqs and impacket, pytest, crypto'
-
-    - run: |
-        git clone --depth=1 -b openssl-3.0.8+quic https://github.com/quictls/openssl
-        cd openssl
-        ./config --prefix=$HOME/all --libdir=$HOME/all/lib
-        make install_sw
-      name: 'install quictls'
-
-    - run: |
-        git clone --depth=1 -b v0.8.0 https://github.com/ngtcp2/nghttp3
-        cd nghttp3
-        autoreconf -fi
-        ./configure --prefix=$HOME/all PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig" --enable-lib-only
-        make install
-      name: 'install nghttp3'
-
-    - run: |
-        git clone --depth=1 -b v0.13.1 https://github.com/ngtcp2/ngtcp2
-        cd ngtcp2
-        autoreconf -fi
-        ./configure ${{ matrix.build.ngtcp2-configure }} --with-openssl
-        make install
-      name: 'install ngtcp2'
-
-    - run: |
-        git clone --depth=1 -b v1.52.0 https://github.com/nghttp2/nghttp2
-        cd nghttp2
-        autoreconf -fi
-        ./configure --prefix=$HOME/all PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig" --enable-http3
-        make install
-      name: 'install nghttp2'
-
-    - run: |
-        git clone --depth=1 -b master https://github.com/icing/mod_h2
-        cd mod_h2
-        autoreconf -fi
-        ./configure PKG_CONFIG_PATH="$HOME/all/lib/pkgconfig"
-        make
-        sudo make install
-      name: 'install mod_h2'
-
-    - uses: actions/checkout@v3
-
-    - run: autoreconf -fi
-      name: 'autoreconf'
-
-    - run: ./configure --with-openssl=$HOME/all ${{ matrix.build.configure }}
-      name: 'configure'
-
-    - run: make V=1
-      name: 'make'
-
-    - run: make V=1 examples
-      name: 'make examples'
-
-    - run: make V=1 -C tests
-      name: 'make tests'
-
-    - run: pytest -v
-      name: 'run pytest'
-      env:
-        TFLAGS: "${{ matrix.build.tflags }}"