cmake: picky-linker fixes for openssl, ZLIB, H3 and more

- fix HTTP/3 support detection with OpenSSL/quictls built with ZLIB.
  (Requires curl be built with ZLIB option also.)

- fix HTTP/3 support detection with OpenSSL/quictls/LibreSSL and `ld`
  linker on Windows.

- fix HTTP/3 support detection with wolfSSL to automatically add
  `ws2_32` to the lib list on Windows. For all linkers.

- reposition ZLIB (and other compression) detection _after_ TLS
  detection, but before calling HTTP/3-support detection via
  `CheckQuicSupportInOpenSSL`.

  May be a regression from ebef55a61df0094b9790710a42f63c48e7de3c13
  May fix #10832 (Reported-by: Micah Snyder)

  This also seems to fix an odd case, where OpenSSL/quictls is correctly
  detected, but its header path is not set while compiling, breaking
  build at `src/curl_ntlm_core.c`. Reason for this remains undiscovered.

- satisfy "picky" linkers such as `ld` with MinGW, that are highly
  sensitive to lib order, by also adding brotli to the beginning of the
  lib list.

- satisfy "picky" linkers by adding certain Windows systems libs to
  the lib list for OpenSSL/LibreSSL. (Might need additional ones for
  other forks, such as `pthread` for BoringSSL.)

Note: It'd make sense to _always_ add `ws2_32`, `crypt32` (except
Windows App targets perhaps?), `bcrypt` (except old-mingw!) on Windows
at this point. They are almost always required, and if some aren't,
they are ignored by the linker with no effect on final binaries.

Closes #10857

1 files changed, 85 insertions, 67 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d2c97cfe1..e3eaad34d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -361,6 +361,19 @@ check_function_exists(gethostname HAVE_GETHOSTNAME)
 if(WIN32)
   check_library_exists_concat("ws2_32" getch        HAVE_LIBWS2_32)
   check_library_exists_concat("winmm"  getch        HAVE_LIBWINMM)
+
+  # Matching logic used for Curl_win32_random()
+  if(MINGW)
+    check_c_source_compiles("
+      #include <_mingw.h>
+      #if defined(__MINGW64_VERSION_MAJOR)
+      #error
+      #endif
+      int main(void) {
+        return 0;
+      }"
+      HAVE_MINGW_ORIGINAL)
+  endif()
 endif()
 
 # check SSL libraries
@@ -431,58 +444,6 @@ if(use_core_foundation)
   list(APPEND CURL_LIBS "-framework CoreFoundation")
 endif()
 
-# Keep compression lib detection before TLS detection, which
-# might depend on it.
-
-set(HAVE_LIBZ OFF)
-set(USE_ZLIB OFF)
-optional_dependency(ZLIB)
-if(ZLIB_FOUND)
-  set(HAVE_LIBZ ON)
-  set(USE_ZLIB ON)
-
-  # Depend on ZLIB via imported targets if supported by the running
-  # version of CMake.  This allows our dependents to get our dependencies
-  # transitively.
-  if(NOT CMAKE_VERSION VERSION_LESS 3.4)
-    list(APPEND CURL_LIBS ZLIB::ZLIB)
-  else()
-    list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
-    include_directories(${ZLIB_INCLUDE_DIRS})
-  endif()
-  list(APPEND CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIRS})
-endif()
-
-option(CURL_BROTLI "Set to ON to enable building curl with brotli support." OFF)
-set(HAVE_BROTLI OFF)
-if(CURL_BROTLI)
-  find_package(Brotli QUIET)
-  if(BROTLI_FOUND)
-    set(HAVE_BROTLI ON)
-    list(APPEND CURL_LIBS ${BROTLI_LIBRARIES})
-    include_directories(${BROTLI_INCLUDE_DIRS})
-    list(APPEND CMAKE_REQUIRED_INCLUDES ${BROTLI_INCLUDE_DIRS})
-  endif()
-endif()
-
-option(CURL_ZSTD "Set to ON to enable building curl with zstd support." OFF)
-set(HAVE_ZSTD OFF)
-if(CURL_ZSTD)
-  find_package(Zstd REQUIRED)
-  if (NOT DEFINED HAVE_ZSTD_CREATEDSTREAM)
-    cmake_push_check_state()
-    set(CMAKE_REQUIRED_INCLUDES ${Zstd_INCLUDE_DIRS})
-    set(CMAKE_REQUIRED_LIBRARIES ${Zstd_LIBRARIES})
-    check_symbol_exists(ZSTD_createDStream "zstd.h" HAVE_ZSTD_CREATEDSTREAM)
-    cmake_pop_check_state()
-  endif()
-  if(Zstd_FOUND AND HAVE_ZSTD_CREATEDSTREAM)
-    set(HAVE_ZSTD ON)
-    list(APPEND CURL_LIBS ${Zstd_LIBRARIES})
-    include_directories(${Zstd_INCLUDE_DIRS})
-  endif()
-endif()
-
 if(CURL_USE_OPENSSL)
   find_package(OpenSSL REQUIRED)
   set(SSL_ENABLED ON)
@@ -498,6 +459,13 @@ if(CURL_USE_OPENSSL)
     include_directories(${OPENSSL_INCLUDE_DIR})
   endif()
 
+  if(WIN32)
+    list(APPEND CURL_LIBS "ws2_32")
+    if(NOT HAVE_MINGW_ORIGINAL)
+      list(APPEND CURL_LIBS "bcrypt")  # for OpenSSL/LibreSSL
+    endif()
+  endif()
+
   set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
   if(NOT DEFINED HAVE_RAND_EGD)
     check_symbol_exists(RAND_egd "${CURL_INCLUDES}" HAVE_RAND_EGD)
@@ -546,6 +514,59 @@ if(CURL_USE_NSS)
   endif()
 endif()
 
+# Keep ZLIB detection after TLS detection,
+# and before calling CheckQuicSupportInOpenSSL.
+
+set(HAVE_LIBZ OFF)
+set(USE_ZLIB OFF)
+optional_dependency(ZLIB)
+if(ZLIB_FOUND)
+  set(HAVE_LIBZ ON)
+  set(USE_ZLIB ON)
+
+  # Depend on ZLIB via imported targets if supported by the running
+  # version of CMake.  This allows our dependents to get our dependencies
+  # transitively.
+  if(NOT CMAKE_VERSION VERSION_LESS 3.4)
+    list(APPEND CURL_LIBS ZLIB::ZLIB)
+  else()
+    list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
+    include_directories(${ZLIB_INCLUDE_DIRS})
+  endif()
+  list(APPEND CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIRS})
+endif()
+
+option(CURL_BROTLI "Set to ON to enable building curl with brotli support." OFF)
+set(HAVE_BROTLI OFF)
+if(CURL_BROTLI)
+  find_package(Brotli QUIET)
+  if(BROTLI_FOUND)
+    set(HAVE_BROTLI ON)
+    list(PREPEND CURL_LIBS ${BROTLI_LIBRARIES})
+    list(APPEND CURL_LIBS ${BROTLI_LIBRARIES})
+    include_directories(${BROTLI_INCLUDE_DIRS})
+    list(APPEND CMAKE_REQUIRED_INCLUDES ${BROTLI_INCLUDE_DIRS})
+  endif()
+endif()
+
+option(CURL_ZSTD "Set to ON to enable building curl with zstd support." OFF)
+set(HAVE_ZSTD OFF)
+if(CURL_ZSTD)
+  find_package(Zstd REQUIRED)
+  if (NOT DEFINED HAVE_ZSTD_CREATEDSTREAM)
+    cmake_push_check_state()
+    set(CMAKE_REQUIRED_INCLUDES ${Zstd_INCLUDE_DIRS})
+    set(CMAKE_REQUIRED_LIBRARIES ${Zstd_LIBRARIES})
+    check_symbol_exists(ZSTD_createDStream "zstd.h" HAVE_ZSTD_CREATEDSTREAM)
+    cmake_pop_check_state()
+  endif()
+  if(Zstd_FOUND AND HAVE_ZSTD_CREATEDSTREAM)
+    set(HAVE_ZSTD ON)
+    list(APPEND CURL_LIBS ${Zstd_LIBRARIES})
+    include_directories(${Zstd_INCLUDE_DIRS})
+  endif()
+endif()
+
 option(USE_NGHTTP2 "Use Nghttp2 library" OFF)
 if(USE_NGHTTP2)
   find_package(NGHTTP2 REQUIRED)
@@ -565,19 +586,29 @@ function(CheckQuicSupportInOpenSSL)
         list(APPEND CMAKE_REQUIRED_LIBRARIES "${ZLIB_LIBRARIES}")
       endif()
       if(WIN32)
-        list(APPEND CMAKE_REQUIRED_LIBRARIES "crypt32")
+        list(APPEND CMAKE_REQUIRED_LIBRARIES "ws2_32" "crypt32")
       endif()
       list(APPEND CMAKE_REQUIRED_DEFINITIONS -DHAVE_UINTPTR_T)  # to pull in stdint.h (as of wolfSSL v5.5.4)
       check_symbol_exists(wolfSSL_set_quic_method "wolfssl/options.h;wolfssl/openssl/ssl.h" HAVE_SSL_CTX_SET_QUIC_METHOD)
     else()
       set(CMAKE_REQUIRED_INCLUDES   "${OPENSSL_INCLUDE_DIR}")
       set(CMAKE_REQUIRED_LIBRARIES  "${OPENSSL_LIBRARIES}")
+      if(HAVE_LIBZ)
+        list(APPEND CMAKE_REQUIRED_INCLUDES  "${ZLIB_INCLUDE_DIRS}")
+        list(APPEND CMAKE_REQUIRED_LIBRARIES "${ZLIB_LIBRARIES}")
+      endif()
+      if(WIN32)
+        list(APPEND CMAKE_REQUIRED_LIBRARIES "ws2_32")
+        if(NOT HAVE_MINGW_ORIGINAL)
+          list(APPEND CMAKE_REQUIRED_LIBRARIES "bcrypt")  # for OpenSSL/LibreSSL
+        endif()
+      endif()
       check_symbol_exists(SSL_CTX_set_quic_method "openssl/ssl.h" HAVE_SSL_CTX_SET_QUIC_METHOD)
     endif()
     cmake_pop_check_state()
   endif()
   if(NOT HAVE_SSL_CTX_SET_QUIC_METHOD)
-    message(FATAL_ERROR "QUIC support is missing in OpenSSL/BoringSSL/wolfSSL. Try setting -DOPENSSL_ROOT_DIR")
+    message(FATAL_ERROR "QUIC support is missing in OpenSSL/LibreSSL/BoringSSL/wolfSSL. Try setting -DOPENSSL_ROOT_DIR")
   endif()
 endfunction()
 
@@ -1312,19 +1343,6 @@ if(WIN32)
     list(APPEND CURL_LIBS "advapi32" "crypt32")
   endif()
 
-  # Matching logic used for Curl_win32_random()
-  if(MINGW)
-    check_c_source_compiles("
-      #include <_mingw.h>
-      #if defined(__MINGW64_VERSION_MAJOR)
-      #error
-      #endif
-      int main(void) {
-        return 0;
-      }"
-      HAVE_MINGW_ORIGINAL)
-  endif()
-
   if(NOT HAVE_MINGW_ORIGINAL)
     list(APPEND CURL_LIBS "bcrypt")
   else()