config: fix building SMB with configure using Win32 Crypto

Align conditions for NTLM features between CMake and configure
builds by differentiating between USE_NTLM and USE_CURL_NTLM_CORE,
just like curl_setup.h does internally to detect support of:

- USE_NTLM: required for NTLM crypto authentication feature
- USE_CURL_NTLM_CORE: required for SMB protocol

Implement USE_WIN32_CRYPTO detection by checking for Crypt functions
in wincrypt.h which are not available in the Windows App environment.

Link advapi32 and crypt32 for Crypto API and Schannel SSL backend.
Fix condition of Schannel SSL backend in CMake build accordingly.

Reviewed-by: Marcel Raad

Closes #6277

4 files changed, 131 insertions, 39 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 23db36f76..4f12c7910 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -310,7 +310,6 @@ check_function_exists(gethostname HAVE_GETHOSTNAME)
 if(WIN32)
   check_library_exists_concat("ws2_32" getch        HAVE_LIBWS2_32)
   check_library_exists_concat("winmm"  getch        HAVE_LIBWINMM)
-  list(APPEND CURL_LIBS "advapi32")
 endif()
 
 # check SSL libraries
@@ -356,7 +355,6 @@ if(CMAKE_USE_SCHANNEL)
   set(SSL_ENABLED ON)
   set(USE_SCHANNEL ON) # Windows native SSL/TLS support
   set(USE_WINDOWS_SSPI ON) # CMAKE_USE_SCHANNEL implies CURL_WINDOWS_SSPI
-  list(APPEND CURL_LIBS "crypt32")
 endif()
 if(CURL_WINDOWS_SSPI)
   set(USE_WINDOWS_SSPI ON)
@@ -505,10 +503,6 @@ if(USE_QUICHE)
   cmake_pop_check_state()
 endif()
 
-if(WIN32)
-  set(USE_WIN32_CRYPTO ON)
-endif()
-
 if(NOT CURL_DISABLE_LDAP)
   if(WIN32)
     option(USE_WIN32_LDAP "Use Windows LDAP implementation" ON)
@@ -874,9 +868,7 @@ if(NOT UNIX)
   check_include_file_concat("winsock.h"      HAVE_WINSOCK_H)
   check_include_file_concat("ws2tcpip.h"     HAVE_WS2TCPIP_H)
   check_include_file_concat("winsock2.h"     HAVE_WINSOCK2_H)
-  if(NOT CURL_WINDOWS_SSPI AND USE_OPENSSL)
-    set(CURL_LIBS ${CURL_LIBS} "crypt32")
-  endif()
+  check_include_file_concat("wincrypt.h"     HAVE_WINCRYPT_H)
 endif()
 
 check_include_file_concat("stdio.h"          HAVE_STDIO_H)
@@ -1252,6 +1244,19 @@ if(WIN32)
 
   # Use the manifest embedded in the Windows Resource
   set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -DCURL_EMBED_MANIFEST")
+
+  # Check if crypto functions in wincrypt.h are actually available
+  if(HAVE_WINCRYPT_H)
+    check_symbol_exists(CryptAcquireContext "${CURL_INCLUDES}" USE_WINCRYPT)
+  endif()
+  if(USE_WINCRYPT)
+    set(USE_WIN32_CRYPTO ON)
+  endif()
+
+  # Link required libraries for USE_WIN32_CRYPTO or USE_SCHANNEL
+  if(USE_WIN32_CRYPTO OR USE_SCHANNEL)
+    list(APPEND CURL_LIBS "advapi32" "crypt32")
+  endif()
 endif()
 
 if(MSVC)
@@ -1333,14 +1338,6 @@ if(BUILD_TESTING)
   add_subdirectory(tests)
 endif()
 
-# NTLM support requires crypto function adaptions from various SSL libs
-# TODO alternative SSL libs tests for SSP1, GNUTLS, NSS
-if(NOT CURL_DISABLE_CRYPTO_AUTH AND (USE_OPENSSL OR USE_DARWINSSL OR USE_MBEDTLS OR USE_WIN32_CRYPTO))
-  set(use_ntlm ON)
-else()
-  set(use_ntlm OFF)
-endif()
-
 # Helper to populate a list (_items) with a label when conditions (the remaining
 # args) are satisfied
 macro(_add_if label)
@@ -1350,6 +1347,13 @@ macro(_add_if label)
   endif()
 endmacro()
 
+# NTLM support requires crypto function adaptions from various SSL libs
+# TODO alternative SSL libs tests for SSP1, GNUTLS, NSS
+if(NOT CURL_DISABLE_CRYPTO_AUTH AND (USE_OPENSSL OR USE_MBEDTLS OR
+                                     USE_DARWINSSL OR USE_WIN32_CRYPTO))
+  set(use_curl_ntlm_core ON)
+endif()
+
 # Clear list and try to detect available features
 set(_items)
 _add_if("SSL"           SSL_ENABLED)
@@ -1373,9 +1377,10 @@ _add_if("Kerberos"      NOT CURL_DISABLE_CRYPTO_AUTH AND
                         (HAVE_GSSAPI OR USE_WINDOWS_SSPI))
 # NTLM support requires crypto function adaptions from various SSL libs
 # TODO alternative SSL libs tests for SSP1, GNUTLS, NSS
-_add_if("NTLM"        use_ntlm OR USE_WINDOWS_SSPI)
+_add_if("NTLM"          (use_curl_ntlm_core OR USE_WINDOWS_SSPI))
 # TODO missing option (autoconf: --enable-ntlm-wb)
-_add_if("NTLM_WB"     use_ntlm AND NOT CURL_DISABLE_HTTP AND NTLM_WB_ENABLED)
+_add_if("NTLM_WB"       (use_curl_ntlm_core OR USE_WINDOWS_SSPI) AND
+                        NOT CURL_DISABLE_HTTP AND NTLM_WB_ENABLED)
 # TODO missing option (--enable-tls-srp), depends on GNUTLS_SRP/OPENSSL_SRP
 _add_if("TLS-SRP"       USE_TLS_SRP)
 # TODO option --with-nghttp2 tests for nghttp2 lib and nghttp2/nghttp2.h header
@@ -1409,8 +1414,10 @@ _add_if("POP3"          NOT CURL_DISABLE_POP3)
 _add_if("POP3S"         NOT CURL_DISABLE_POP3 AND SSL_ENABLED)
 _add_if("IMAP"          NOT CURL_DISABLE_IMAP)
 _add_if("IMAPS"         NOT CURL_DISABLE_IMAP AND SSL_ENABLED)
-_add_if("SMB"           NOT CURL_DISABLE_SMB AND use_ntlm)
-_add_if("SMBS"          NOT CURL_DISABLE_SMB AND SSL_ENABLED AND use_ntlm)
+_add_if("SMB"           NOT CURL_DISABLE_SMB AND
+                        use_curl_ntlm_core AND (SIZEOF_CURL_OFF_T GREATER 4))
+_add_if("SMBS"          NOT CURL_DISABLE_SMB AND SSL_ENABLED AND
+                        use_curl_ntlm_core AND (SIZEOF_CURL_OFF_T GREATER 4))
 _add_if("SMTP"          NOT CURL_DISABLE_SMTP)
 _add_if("SMTPS"         NOT CURL_DISABLE_SMTP AND SSL_ENABLED)
 _add_if("SCP"           USE_LIBSSH2 OR USE_LIBSSH)
@@ -1426,7 +1433,7 @@ message(STATUS "Enabled protocols: ${SUPPORT_PROTOCOLS}")
 
 # Clear list and collect SSL backends
 set(_items)
-_add_if("Schannel"         SSL_ENABLED AND USE_WINDOWS_SSPI)
+_add_if("Schannel"         SSL_ENABLED AND USE_SCHANNEL)
 _add_if("OpenSSL"          SSL_ENABLED AND USE_OPENSSL)
 _add_if("Secure Transport" SSL_ENABLED AND USE_SECTRANSP)
 _add_if("mbedTLS"          SSL_ENABLED AND USE_MBEDTLS)
diff --git a/acinclude.m4 b/acinclude.m4
index c65e1ab62..de88852b3 100644
--- a/acinclude.m4
+++ b/acinclude.m4
@@ -347,6 +347,39 @@ AC_DEFUN([CURL_CHECK_HEADER_WS2TCPIP], [
 ])
 
 
+dnl CURL_CHECK_HEADER_WINCRYPT
+dnl -------------------------------------------------
+dnl Check for compilable and valid wincrypt.h header
+
+AC_DEFUN([CURL_CHECK_HEADER_WINCRYPT], [
+  AC_REQUIRE([CURL_CHECK_HEADER_WINDOWS])dnl
+  AC_CACHE_CHECK([for wincrypt.h], [curl_cv_header_wincrypt_h], [
+    AC_COMPILE_IFELSE([
+      AC_LANG_PROGRAM([[
+#undef inline
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+#include <wincrypt.h>
+      ]],[[
+        int dummy=2*PROV_RSA_FULL;
+      ]])
+    ],[
+      curl_cv_header_wincrypt_h="yes"
+    ],[
+      curl_cv_header_wincrypt_h="no"
+    ])
+  ])
+  case "$curl_cv_header_wincrypt_h" in
+    yes)
+      AC_DEFINE_UNQUOTED(HAVE_WINCRYPT_H, 1,
+        [Define to 1 if you have the wincrypt.h header file.])
+      ;;
+  esac
+])
+
+
 dnl CURL_CHECK_HEADER_WINLDAP
 dnl -------------------------------------------------
 dnl Check for compilable and valid winldap.h header
@@ -2353,11 +2386,54 @@ AC_DEFUN([CURL_CHECK_WIN32_LARGEFILE], [
       AC_MSG_RESULT([yes (large file enabled)])
       AC_DEFINE_UNQUOTED(USE_WIN32_LARGE_FILES, 1,
         [Define to 1 if you are building a Windows target with large file support.])
+      AC_SUBST(USE_WIN32_LARGE_FILES, [1])
       ;;
     win32_small_files)
       AC_MSG_RESULT([yes (large file disabled)])
       AC_DEFINE_UNQUOTED(USE_WIN32_SMALL_FILES, 1,
         [Define to 1 if you are building a Windows target without large file support.])
+      AC_SUBST(USE_WIN32_SMALL_FILES, [1])
+      ;;
+    *)
+      AC_MSG_RESULT([no])
+      ;;
+  esac
+])
+
+dnl CURL_CHECK_WIN32_CRYPTO
+dnl -------------------------------------------------
+dnl Check if curl's WIN32 crypto lib can be used
+
+AC_DEFUN([CURL_CHECK_WIN32_CRYPTO], [
+  AC_REQUIRE([CURL_CHECK_HEADER_WINCRYPT])dnl
+  AC_MSG_CHECKING([whether build target supports WIN32 crypto API])
+  curl_win32_crypto_api="no"
+  if test "$curl_cv_header_wincrypt_h" = "yes"; then
+    AC_COMPILE_IFELSE([
+      AC_LANG_PROGRAM([[
+#undef inline
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+#include <wincrypt.h>
+      ]],[[
+        HCRYPTPROV hCryptProv;
+        if(CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL,
+                               CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) {
+          CryptReleaseContext(hCryptProv, 0);
+        }
+      ]])
+    ],[
+      curl_win32_crypto_api="yes"
+    ])
+  fi
+  case "$curl_win32_crypto_api" in
+    yes)
+      AC_MSG_RESULT([yes])
+      AC_DEFINE_UNQUOTED(USE_WIN32_CRYPTO, 1,
+        [Define to 1 if you are building a Windows target with crypto API support.])
+      AC_SUBST(USE_WIN32_CRYPTO, [1])
       ;;
     *)
       AC_MSG_RESULT([no])
diff --git a/configure.ac b/configure.ac
index a16804419..42e6838df 100755
--- a/configure.ac
+++ b/configure.ac
@@ -357,6 +357,7 @@ case X-"$curl_cv_native_windows" in
     CURL_CHECK_HEADER_WINSOCK
     CURL_CHECK_HEADER_WINSOCK2
     CURL_CHECK_HEADER_WS2TCPIP
+    CURL_CHECK_HEADER_WINCRYPT
     CURL_CHECK_HEADER_WINLDAP
     CURL_CHECK_HEADER_WINBER
     ;;
@@ -364,11 +365,13 @@ case X-"$curl_cv_native_windows" in
     curl_cv_header_winsock_h="no"
     curl_cv_header_winsock2_h="no"
     curl_cv_header_ws2tcpip_h="no"
+    curl_cv_header_wincrypt_h="no"
     curl_cv_header_winldap_h="no"
     curl_cv_header_winber_h="no"
     ;;
 esac
 CURL_CHECK_WIN32_LARGEFILE
+CURL_CHECK_WIN32_CRYPTO
 
 CURL_MAC_CFLAGS
 CURL_SUPPORTS_BUILTIN_AVAILABLE
@@ -1648,7 +1651,6 @@ if test -z "$ssl_backends" -o "x$OPT_SCHANNEL" != xno; then
     AC_DEFINE(USE_WINDOWS_SSPI, 1, [to enable SSPI support])
     AC_SUBST(USE_WINDOWS_SSPI, [1])
     curl_sspi_msg="enabled"
-    LIBS="-lcrypt32 $LIBS"
   else
     AC_MSG_RESULT(no)
   fi
@@ -1657,6 +1659,11 @@ else
   AC_MSG_RESULT(no)
 fi
 
+dnl link required libraries for USE_WIN32_CRYPTO or USE_SCHANNEL
+if test "x$USE_WIN32_CRYPTO" = "x1" -o "x$USE_SCHANNEL" = "x1"; then
+  LIBS="-ladvapi32 -lcrypt32 $LIBS"
+fi
+
 OPT_SECURETRANSPORT=no
 AC_ARG_WITH(darwinssl,dnl
 AC_HELP_STRING([--with-darwinssl],[enable Apple OS native SSL/TLS])
@@ -5222,17 +5229,23 @@ if test "x$CURL_DISABLE_CRYPTO_AUTH" != "x1" -a \
   SUPPORT_FEATURES="$SUPPORT_FEATURES Kerberos"
 fi
 
-if test "x$CURL_DISABLE_CRYPTO_AUTH" != "x1"; then
-  if test "x$OPENSSL_ENABLED" = "x1" -o "x$USE_WINDOWS_SSPI" = "x1" \
-      -o "x$GNUTLS_ENABLED" = "x1" -o "x$MBEDTLS_ENABLED" = "x1" \
-      -o "x$NSS_ENABLED" = "x1" -o "x$SECURETRANSPORT_ENABLED" = "x1" \
-      -o "x$WOLFSSL_NTLM" = "x1"; then
-    SUPPORT_FEATURES="$SUPPORT_FEATURES NTLM"
+if test "x$CURL_DISABLE_CRYPTO_AUTH" != "x1" -a \
+    \( "x$OPENSSL_ENABLED" = "x1" -o "x$MBEDTLS_ENABLED" = "x1" \
+      -o "x$GNUTLS_ENABLED" = "x1" -o "x$NSS_ENABLED" = "x1" \
+      -o "x$SECURETRANSPORT_ENABLED" = "x1" \
+      -o "x$USE_WIN32_CRYPTO" = "x1" \
+      -o "x$WOLFSSL_NTLM" = "x1" \); then
+  use_curl_ntlm_core=yes
+else
+  use_curl_ntlm_core=no
+fi
+if test "x$use_curl_ntlm_core" = "xyes" \
+    -o "x$USE_WINDOWS_SSPI" = "x1"; then
+  SUPPORT_FEATURES="$SUPPORT_FEATURES NTLM"
 
-    if test "x$CURL_DISABLE_HTTP" != "x1" -a \
-        "x$NTLM_WB_ENABLED" = "x1"; then
-      SUPPORT_FEATURES="$SUPPORT_FEATURES NTLM_WB"
-    fi
+  if test "x$CURL_DISABLE_HTTP" != "x1" -a \
+      "x$NTLM_WB_ENABLED" = "x1"; then
+    SUPPORT_FEATURES="$SUPPORT_FEATURES NTLM_WB"
   fi
 fi
 
@@ -5333,11 +5346,7 @@ if test "x$CURL_DISABLE_IMAP" != "x1"; then
   fi
 fi
 if test "x$CURL_DISABLE_SMB" != "x1" \
-    -a "x$CURL_DISABLE_CRYPTO_AUTH" != "x1" \
-    -a \( "x$OPENSSL_ENABLED" = "x1" \
-      -o "x$GNUTLS_ENABLED" = "x1" -o "x$MBEDTLS_ENABLED" = "x1" \
-      -o "x$NSS_ENABLED" = "x1" -o "x$SECURETRANSPORT_ENABLED" = "x1" \
-      -o "x$WOLFSSL_NTLM" = "x1" \); then
+    -a "x$use_curl_ntlm_core" = "xyes"; then
   SUPPORT_PROTOCOLS="$SUPPORT_PROTOCOLS SMB"
   if test "x$SSL_ENABLED" = "x1"; then
     SUPPORT_PROTOCOLS="$SUPPORT_PROTOCOLS SMBS"
diff --git a/lib/curl_config.h.cmake b/lib/curl_config.h.cmake
index 8ced43672..01c4ded1f 100644
--- a/lib/curl_config.h.cmake
+++ b/lib/curl_config.h.cmake
@@ -98,7 +98,7 @@
 #endif
 
 /* Allow SMB to work on Windows */
-#cmakedefine USE_WIN32_CRYPTO
+#cmakedefine USE_WIN32_CRYPTO 1
 
 /* Use Windows LDAP implementation */
 #cmakedefine USE_WIN32_LDAP 1