transport: allow cred/cert callbacks to return >0ethomson/defer_cert_cred_cb_0_27

Allow credential and certificate checking callbacks to return > 0,
indicating that they did not want to act.  This was documented in the
credential acquisition callback already, but only loosely implemented.
Expand this to being supported in the http and ssh callbacks.
Additionally, enable the same mechanism for certificate validation.

This is useful to disambiguate any meaning for returning > 0 in the
publicly exposed credential and certificate functions
(`git_transport_smart_certificate_check`) and
(`git_transport_smart_credentials`).  But it may be more generally
useful for callers to be able to defer back to libgit2.

4 files changed, 29 insertions, 6 deletions

diff --git a/include/git2/types.h b/include/git2/types.h
index 8d9a94710..8183b6c4e 100644
--- a/include/git2/types.h
+++ b/include/git2/types.h
@@ -330,6 +330,9 @@ typedef struct {
  * this certificate is valid
  * @param host Hostname of the host libgit2 connected to
  * @param payload Payload provided by the caller
+ * @return 0 to proceed with the connection, < 0 to fail the connection
+ *         or > 0 to indicate that the callback refused to act and that
+ *         the existing validity determination should be honored
  */
 typedef int (*git_transport_certificate_check_cb)(git_cert *cert, int valid, const char *host, void *payload);
 
diff --git a/src/transports/http.c b/src/transports/http.c
index 22b8bb864..14bd38b8d 100644
--- a/src/transports/http.c
+++ b/src/transports/http.c
@@ -367,11 +367,15 @@ static int on_headers_complete(http_parser *parser)
 								  allowed_auth_types,
 								  t->owner->cred_acquire_payload);
 
-				if (error == GIT_PASSTHROUGH) {
-					no_callback = 1;
-				} else if (error < 0) {
+				/* treat GIT_PASSTHROUGH as if callback isn't set */
+				if (error == GIT_PASSTHROUGH)
+					error = 1;
+
+				if (error < 0) {
 					t->error = error;
 					return t->parse_error = PARSE_ERROR_EXT;
+				} else if (error > 0) {
+					no_callback = 1;
 				} else {
 					assert(t->cred);
 
@@ -634,6 +638,9 @@ static int http_connect(http_subtransport *t)
 		giterr_clear();
 		error = t->owner->certificate_check_cb(cert, is_valid, t->connection_data.host, t->owner->message_cb_payload);
 
+		if (error > 0)
+			error = is_valid ? 0 : GIT_ECERTIFICATE;
+
 		if (error < 0) {
 			if (!giterr_last())
 				giterr_set(GITERR_NET, "user cancelled certificate check");
diff --git a/src/transports/ssh.c b/src/transports/ssh.c
index 23c643346..614514337 100644
--- a/src/transports/ssh.c
+++ b/src/transports/ssh.c
@@ -448,9 +448,12 @@ static int request_creds(git_cred **out, ssh_subtransport *t, const char *user,
 						  t->owner->cred_acquire_payload);
 
 		if (error == GIT_PASSTHROUGH)
-			no_callback = 1;
-		else if (error < 0)
+			error = 1;
+
+		if (error < 0)
 			return error;
+		else if (error > 0)
+			no_callback = 1;
 		else if (!cred) {
 			giterr_set(GITERR_SSH, "callback failed to initialize SSH credentials");
 			return -1;
@@ -584,6 +587,11 @@ post_extract:
 		cert_ptr = &cert;
 
 		error = t->owner->certificate_check_cb((git_cert *) cert_ptr, 0, host, t->owner->message_cb_payload);
+
+		/* If the callback chose not to act, fail this hostkey */
+		if (error > 0)
+			error = GIT_ECERTIFICATE;
+
 		if (error < 0) {
 			if (!giterr_last())
 				giterr_set(GITERR_NET, "user cancelled hostkey check");
diff --git a/src/transports/winhttp.c b/src/transports/winhttp.c
index c3c18a80a..56e25b43c 100644
--- a/src/transports/winhttp.c
+++ b/src/transports/winhttp.c
@@ -226,7 +226,9 @@ static int fallback_cred_acquire_cb(
 			return -1;
 		}
 
-		if (SUCCEEDED(CoInitializeEx(NULL, COINIT_MULTITHREADED))) {
+		hCoInitResult = CoInitializeEx(NULL, COINIT_MULTITHREADED);
+
+		if (SUCCEEDED(hCoInitResult) || hCoInitResult == RPC_E_CHANGED_MODE) {
 			IInternetSecurityManager* pISM;
 
 			/* And if the target URI is in the My Computer, Intranet, or Trusted zones */
@@ -290,6 +292,9 @@ static int certificate_check(winhttp_stream *s, int valid)
 	error = t->owner->certificate_check_cb((git_cert *) &cert, valid, t->connection_data.host, t->owner->message_cb_payload);
 	CertFreeCertificateContext(cert_ctx);
 
+	if (error > 0)
+		error = valid ? 0 : GIT_ECERTIFICATE;
+
 	if (error < 0 && !giterr_last())
 		giterr_set(GITERR_NET, "user cancelled certificate check");