Merge branch 'tmp-false-start-clarifications' into 'master'

Fixed operation under multiple threads

Closes #713

See merge request gnutls/gnutls!935

15 files changed, 627 insertions, 151 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cc67461034..9da0e870c0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -218,11 +218,11 @@ asan.Fedora.x86_64:
   - CFLAGS="-fsanitize=address -g -O2" CXXFLAGS=$CFLAGS LDFLAGS="-static-libasan"
     dash ./configure --disable-gcc-warnings --disable-doc --cache-file cache/config.cache --disable-non-suiteb-curves --disable-guile
   - make -j$(nproc)
-  - LSAN_OPTIONS="suppressions=$(pwd)/fuzz/lsan.supp" make check -j$(nproc)
-  - LSAN_OPTIONS="suppressions=$(pwd)/fuzz/lsan.supp" make -C fuzz check -j$(nproc) GNUTLS_CPUID_OVERRIDE=0x1
-  - LSAN_OPTIONS="suppressions=$(pwd)/fuzz/lsan.supp" make -C fuzz check -j$(nproc) GNUTLS_CPUID_OVERRIDE=0x2
-  - LSAN_OPTIONS="suppressions=$(pwd)/fuzz/lsan.supp" make -C fuzz check -j$(nproc) GNUTLS_CPUID_OVERRIDE=0x4
-  - LSAN_OPTIONS="suppressions=$(pwd)/fuzz/lsan.supp" make -C fuzz check -j$(nproc) GNUTLS_CPUID_OVERRIDE=0x8
+  - LSAN_OPTIONS="suppressions=$(pwd)/devel/lsan.supp" make check -j$(nproc)
+  - LSAN_OPTIONS="suppressions=$(pwd)/devel/lsan.supp" make -C fuzz check -j$(nproc) GNUTLS_CPUID_OVERRIDE=0x1
+  - LSAN_OPTIONS="suppressions=$(pwd)/devel/lsan.supp" make -C fuzz check -j$(nproc) GNUTLS_CPUID_OVERRIDE=0x2
+  - LSAN_OPTIONS="suppressions=$(pwd)/devel/lsan.supp" make -C fuzz check -j$(nproc) GNUTLS_CPUID_OVERRIDE=0x4
+  - LSAN_OPTIONS="suppressions=$(pwd)/devel/lsan.supp" make -C fuzz check -j$(nproc) GNUTLS_CPUID_OVERRIDE=0x8
   - CFLAGS="-fsanitize=address -g -O2" CXXFLAGS=$CFLAGS LDFLAGS="-static-libasan"
     dash ./configure --cache-file cache/config.cache --disable-doc --with-system-priority-file=/etc/crypto-policies/back-ends/gnutls.config --with-default-priority-string=@SYSTEM --with-default-trust-store-pkcs11="pkcs11:" --disable-guile
   - make -j$(nproc)
@@ -242,6 +242,30 @@ asan.Fedora.x86_64:
       - tests/suite/*/*.log
   retry: 1
 
+threadsan.Fedora.x86_64:
+  stage: stage1-testing
+  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
+  script:
+  - ./bootstrap
+  - CFLAGS="-fsanitize=thread -g -O2" CXXFLAGS=$CFLAGS
+    dash ./configure --disable-gcc-warnings --disable-doc --cache-file cache/config.cache --disable-non-suiteb-curves --disable-guile --enable-fips140-mode
+  - make -j$(nproc)
+  - make -C tests check -j$(nproc) SUBDIRS=. TESTS="tls-pthread dtls-pthread fips-mode-pthread rng-pthread" TSAN_OPTIONS="suppressions=$(pwd)/devel/tsan.supp" GNUTLS_SKIP_FIPS_INTEGRITY_CHECKS=1 GNUTLS_FORCE_FIPS_MODE=1
+  tags:
+  - shared
+  except:
+  - tags
+  artifacts:
+    expire_in: 1 week
+    when: on_failure
+    paths:
+      - ./*.log
+      - fuzz/*.log
+      - tests/*.log
+      - tests/*/*.log
+      - tests/suite/*/*.log
+  retry: 1
+
 static-analyzers.Fedora.x86_64:
   stage: stage1-testing
   image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
diff --git a/NEWS b/NEWS
index 63b99641d7..4d38bd9a0e 100644
--- a/NEWS
+++ b/NEWS
@@ -26,6 +26,9 @@ See the end for copying conditions.
    an inappropriate for TLS1.3 certificate is seen on the credentials structure
    GnuTLS will disable TLS1.3 support for that session (#690).
 
+** libgnutls: fixed issue preventing sending and receiving from different
+   threads when false start was enabled (#713).
+
 ** libgnutls: enforce the equality of the two signature parameters fields in
    a certificate. We were already enforcing the signature algorithm, but there
    was a bug in parameter checking code.
diff --git a/fuzz/lsan.supp b/devel/lsan.supp
index 05541f71a1..05541f71a1 100644
--- a/fuzz/lsan.supp
+++ b/devel/lsan.supp
diff --git a/devel/tsan.supp b/devel/tsan.supp
new file mode 100644
index 0000000000..20a835b0af
--- /dev/null
+++ b/devel/tsan.supp
@@ -0,0 +1,9 @@
+# the following is an inherent issue on gnutls_record_get_direction()
+race:_gnutls_io_write_buffered
+race:_gnutls_io_write_flush
+
+# the following is an inherent issue on gnutls_transport_set_errno()
+race:reset_errno
+
+# this updates the recv_state to states that are harmless
+race:check_session_status
diff --git a/doc/cha-gtls-app.texi b/doc/cha-gtls-app.texi
index 9831db2da4..21df8e5fb3 100644
--- a/doc/cha-gtls-app.texi
+++ b/doc/cha-gtls-app.texi
@@ -234,11 +234,21 @@ Read-only access to objects, for example the credentials holding structures,
 is also thread-safe. 
 
 A @code{gnutls_session_t} object could also be shared by two threads, one sending,
-the other receiving. In that case however, care must be taken during key
-updates and re-handshakes to be handled only by a single thread. The termination of a session 
-should be handled, either by a single thread being active, or by the sender thread 
-using @funcref{gnutls_bye} with @code{GNUTLS_SHUT_WR} and the receiving thread 
-waiting for a return value of zero.
+the other receiving. However, care must be taken on the following use cases:
+@itemize
+@item The re-handshake process in TLS 1.2 or earlier must be handled only in
+a single thread and no other thread may be performing any operation.
+@item The flag @code{GNUTLS_AUTO_REAUTH} cannot be used safely in this mode of operation.
+@item Any other operation which may send or receive data, like key update (c.f.,
+@funcref{gnutls_session_key_update}), must not be performed while threads
+are receiving or writing.
+@item The termination of a session should be handled, either by a single thread being
+active, or by the sender thread using @funcref{gnutls_bye} with @code{GNUTLS_SHUT_WR}
+and the receiving thread waiting for a return value of zero (or timeout on
+certain servers which do not respond).
+@item The functions @funcref{gnutls_transport_set_errno} and @funcref{gnutls_record_get_direction}
+should not be relied during parallel operation.
+@end itemize
 
 For several aspects of the library (e.g., the random generator, PKCS#11
 operations), the library may utilize mutex locks (e.g., pthreads on GNU/Linux and CriticalSection on Windows)
diff --git a/lib/constate.c b/lib/constate.c
index e6ed8a3532..f2c9b839c1 100644
--- a/lib/constate.c
+++ b/lib/constate.c
@@ -39,6 +39,7 @@
 #include "secrets.h"
 #include "handshake.h"
 #include "crypto-api.h"
+#include "locks.h"
 
 static const char keyexp[] = "key expansion";
 static const int keyexp_length = sizeof(keyexp) - 1;
@@ -906,18 +907,28 @@ _gnutls_epoch_get(gnutls_session_t session, unsigned int epoch_rel,
 	record_parameters_st **params;
 	int ret;
 
+	gnutls_mutex_lock(&session->internals.epoch_lock);
+
 	ret = epoch_resolve(session, epoch_rel, &epoch);
-	if (ret < 0)
-		return gnutls_assert_val(ret);
+	if (ret < 0) {
+		gnutls_assert();
+		goto cleanup;
+	}
 
 	params = epoch_get_slot(session, epoch);
-	if (params == NULL || *params == NULL)
-		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+	if (params == NULL || *params == NULL) {
+		ret = gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
+		goto cleanup;
+	}
 
 	if (params_out)
 		*params_out = *params;
 
-	return 0;
+	ret = 0;
+
+cleanup:
+	gnutls_mutex_unlock(&session->internals.epoch_lock);
+	return ret;
 }
 
 /* Ensures that the next epoch is setup. When an epoch will null ciphers
@@ -1008,6 +1019,8 @@ void _gnutls_epoch_gc(gnutls_session_t session)
 
 	_gnutls_record_log("REC[%p]: Start of epoch cleanup\n", session);
 
+	gnutls_mutex_lock(&session->internals.epoch_lock);
+
 	/* Free all dead cipher state */
 	for (i = 0; i < MAX_EPOCH_INDEX; i++) {
 		if (session->record_parameters[i] != NULL) {
@@ -1050,6 +1063,8 @@ void _gnutls_epoch_gc(gnutls_session_t session)
 		session->security_parameters.epoch_min =
 		    session->record_parameters[0]->epoch;
 
+	gnutls_mutex_unlock(&session->internals.epoch_lock);
+
 	_gnutls_record_log("REC[%p]: End of epoch cleanup\n", session);
 }
 
diff --git a/lib/gnutls_int.h b/lib/gnutls_int.h
index 2352299cd8..4547ead0a4 100644
--- a/lib/gnutls_int.h
+++ b/lib/gnutls_int.h
@@ -1281,6 +1281,9 @@ typedef struct {
 
 	/* A handshake process has been completed */
 	bool initial_negotiation_completed;
+	void *post_negotiation_lock; /* protects access to the variable above
+				      * in the cases where negotiation is incomplete
+				      * after gnutls_handshake() - early/false start */
 
 	/* The type of transport protocol; stream or datagram */
 	transport_t transport;
@@ -1466,6 +1469,10 @@ typedef struct {
 	/* anti-replay measure for 0-RTT mode */
 	gnutls_anti_replay_t anti_replay;
 
+	/* Protects _gnutls_epoch_gc() from _gnutls_epoch_get(); these may be
+	 * called in parallel when false start is used and false start is used. */
+	void *epoch_lock;
+
 	/* If you add anything here, check _gnutls_handshake_internal_state_clear().
 	 */
 } internals_st;
diff --git a/lib/handshake-tls13.c b/lib/handshake-tls13.c
index fedef6cbbc..8f9ec7842c 100644
--- a/lib/handshake-tls13.c
+++ b/lib/handshake-tls13.c
@@ -56,6 +56,7 @@
 #include "tls13/finished.h"
 #include "tls13/key_update.h"
 #include "ext/pre_shared_key.h"
+#include "locks.h"
 
 static int generate_rms_keys(gnutls_session_t session);
 static int generate_hs_traffic_keys(gnutls_session_t session);
@@ -202,8 +203,8 @@ int _gnutls13_handshake_client(gnutls_session_t session)
 		return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
 	}
 
-
-	/* explicitly reset any false start flags */
+	/* no lock of post_negotiation_lock is required here as this is not run
+	 * after handshake */
 	session->internals.recv_state = RECV_STATE_0;
 	session->internals.initial_negotiation_completed = 1;
 
@@ -577,9 +578,11 @@ int _gnutls13_handshake_server(gnutls_session_t session)
 		return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
 	}
 
-	/* explicitly reset any false start flags */
+	/* explicitly reset any early start flags */
+	gnutls_mutex_lock(&session->internals.post_negotiation_lock);
 	session->internals.recv_state = RECV_STATE_0;
 	session->internals.initial_negotiation_completed = 1;
+	gnutls_mutex_unlock(&session->internals.post_negotiation_lock);
 
 	SAVE_TRANSCRIPT;
 
diff --git a/lib/handshake.c b/lib/handshake.c
index 481210ebc0..c6762076aa 100644
--- a/lib/handshake.c
+++ b/lib/handshake.c
@@ -55,6 +55,7 @@
 #include <dtls.h>
 #include "secrets.h"
 #include "tls13/session_ticket.h"
+#include "locks.h"
 
 #define TRUE 1
 #define FALSE 0
@@ -1008,8 +1009,6 @@ int _gnutls_recv_finished(gnutls_session_t session)
 	}
 
 
-	session->internals.initial_negotiation_completed = 1;
-
       cleanup:
 	_gnutls_buffer_clear(&buf);
 
@@ -3144,7 +3143,10 @@ static int handshake_client(gnutls_session_t session)
 	}
 
 	/* explicitly reset any false start flags */
+	gnutls_mutex_lock(&session->internals.post_negotiation_lock);
+	session->internals.initial_negotiation_completed = 1;
 	session->internals.recv_state = RECV_STATE_0;
+	gnutls_mutex_unlock(&session->internals.post_negotiation_lock);
 
 	return 0;
 }
@@ -3363,7 +3365,6 @@ static int recv_handshake_final(gnutls_session_t session, int init)
 		break;
 	}
 
-
 	return 0;
 }
 
@@ -3560,6 +3561,10 @@ static int handshake_server(gnutls_session_t session)
 		break;
 	}
 
+	/* no lock of post_negotiation_lock is required here as this is not run
+	 * after handshake */
+	session->internals.initial_negotiation_completed = 1;
+
 	return _gnutls_check_id_for_change(session);
 }
 
diff --git a/lib/record.c b/lib/record.c
index 08aad540db..e7f8e4f01a 100644
--- a/lib/record.c
+++ b/lib/record.c
@@ -53,6 +53,7 @@
 #include <dh.h>
 #include <random.h>
 #include <xsize.h>
+#include "locks.h"
 
 struct tls_record_st {
 	uint16_t header_size;
@@ -1694,8 +1695,7 @@ check_session_status(gnutls_session_t session, unsigned ms)
 		    !(session->internals.flags & GNUTLS_ENABLE_FALSE_START))
 			return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
 
-		/* Attempt to complete handshake */
-
+		/* Attempt to complete handshake - we only need to receive */
 		session->internals.recv_state = RECV_STATE_FALSE_START_HANDLING;
 		ret = gnutls_handshake(session);
 		if (ret < 0) {
@@ -1714,7 +1714,7 @@ check_session_status(gnutls_session_t session, unsigned ms)
 		    !(session->internals.flags & GNUTLS_ENABLE_EARLY_START))
 			return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
 
-		/* Attempt to complete handshake */
+		/* Attempt to complete handshake - we only need to receive */
 		session->internals.recv_state = RECV_STATE_EARLY_START_HANDLING;
 		ret = gnutls_handshake(session);
 		if (ret < 0) {
@@ -1987,12 +1987,23 @@ gnutls_record_send2(gnutls_session_t session, const void *data,
 
 	if (unlikely(!session->internals.initial_negotiation_completed)) {
 		/* this is to protect buggy applications from sending unencrypted
-		 * data. We allow sending however, if we are in false start handshake
-		 * state. */
-		if (session->internals.recv_state != RECV_STATE_FALSE_START &&
+		 * data. We allow sending however, if we are in false or early start
+		 * handshake state. */
+		gnutls_mutex_lock(&session->internals.post_negotiation_lock);
+
+		/* we intentionally re-check the initial_negotation_completed variable
+		 * to avoid locking during normal operation of gnutls_record_send2() */
+		if (!session->internals.initial_negotiation_completed &&
+		    session->internals.recv_state != RECV_STATE_FALSE_START &&
+		    session->internals.recv_state != RECV_STATE_FALSE_START_HANDLING &&
 		    session->internals.recv_state != RECV_STATE_EARLY_START &&
-		    !(session->internals.hsk_flags & HSK_EARLY_DATA_IN_FLIGHT))
+		    session->internals.recv_state != RECV_STATE_EARLY_START_HANDLING &&
+		    !(session->internals.hsk_flags & HSK_EARLY_DATA_IN_FLIGHT)) {
+
+			gnutls_mutex_unlock(&session->internals.post_negotiation_lock);
 			return gnutls_assert_val(GNUTLS_E_UNAVAILABLE_DURING_HANDSHAKE);
+		}
+		gnutls_mutex_unlock(&session->internals.post_negotiation_lock);
 	}
 
 	if (unlikely(!vers))
diff --git a/lib/state.c b/lib/state.c
index 540a83c7b8..a8ff0d81f0 100644
--- a/lib/state.c
+++ b/lib/state.c
@@ -53,6 +53,7 @@
 #include "dtls.h"
 #include "tls13/session_ticket.h"
 #include "ext/cert_types.h"
+#include "locks.h"
 
 /* to be used by supplemental data support to disable TLS1.3
  * when supplemental data have been globally registered */
@@ -451,8 +452,25 @@ int gnutls_init(gnutls_session_t * session, unsigned int flags)
 	if (*session == NULL)
 		return GNUTLS_E_MEMORY_ERROR;
 
+	ret = gnutls_mutex_init(&(*session)->internals.post_negotiation_lock);
+	if (ret < 0) {
+		gnutls_assert();
+		gnutls_free(*session);
+		return ret;
+	}
+
+	ret = gnutls_mutex_init(&(*session)->internals.epoch_lock);
+	if (ret < 0) {
+		gnutls_assert();
+		gnutls_mutex_deinit(&(*session)->internals.post_negotiation_lock);
+		gnutls_free(*session);
+		return ret;
+	}
+
 	ret = _gnutls_epoch_setup_next(*session, 1, NULL);
 	if (ret < 0) {
+		gnutls_mutex_deinit(&(*session)->internals.post_negotiation_lock);
+		gnutls_mutex_deinit(&(*session)->internals.epoch_lock);
 		gnutls_free(*session);
 		return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
 	}
@@ -640,6 +658,9 @@ void gnutls_deinit(gnutls_session_t session)
 	/* overwrite any temp TLS1.3 keys */
 	gnutls_memset(&session->key.proto, 0, sizeof(session->key.proto));
 
+	gnutls_mutex_deinit(&session->internals.post_negotiation_lock);
+	gnutls_mutex_deinit(&session->internals.epoch_lock);
+
 	gnutls_free(session);
 }
 
@@ -1058,7 +1079,7 @@ void
  * interrupted GnuTLS function.
  *
  * This function's output is unreliable if you are using the same
- * @session in different threads, for sending and receiving.
+ * @session in different threads for sending and receiving.
  *
  * Returns: 0 if interrupted while trying to read data, or 1 while trying to write data.
  **/
diff --git a/lib/system_override.c b/lib/system_override.c
index fdf766509c..3c4805bc56 100644
--- a/lib/system_override.c
+++ b/lib/system_override.c
@@ -55,6 +55,9 @@
  * variable that is used by GnuTLS (e.g., the application is linked to
  * msvcr71.dll and gnutls is linked to msvcrt.dll).
  *
+ * This function is unreliable if you are using the same
+ * @session in different threads for sending and receiving.
+ *
  **/
 void gnutls_transport_set_errno(gnutls_session_t session, int err)
 {
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 0c433b0370..bc01e25fb5 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -147,7 +147,7 @@ ctests += mini-record-2 simple gnutls_hmac_fast set_pkcs12_cred cert certuniquei
 	 tls13-rehandshake-cert gnutls_ext_raw_parse handshake-large-cert \
 	 x509cert x509cert-tl infoaccess mini-dtls-hello-verify sign-verify-ed25519-rfc8080 \
 	 trustdb-tofu dtls-rehandshake-anon mini-alpn mini-dtls-large \
-	 mini-termination mini-x509-cas mini-x509-2 pkcs12_simple \
+	 mini-termination mini-x509-cas mini-x509-2 pkcs12_simple tls-pthread \
 	 mini-emsgsize-dtls chainverify-unsorted mini-overhead tls12-ffdhe \
 	 mini-dtls-heartbeat mini-x509-callbacks key-openssl priorities priorities-groups	\
 	 gnutls_x509_privkey_import gnutls_x509_crt_list_import \
@@ -163,7 +163,7 @@ ctests += mini-record-2 simple gnutls_hmac_fast set_pkcs12_cred cert certuniquei
 	 mini-chain-unsorted x509-verify-with-crl mini-dtls-mtu privkey-verify-broken \
 	 mini-dtls-record-asym key-import-export priority-set priority-set2 \
 	 pubkey-import-export sign-is-secure spki spki-abstract rsa-rsa-pss \
-	 mini-dtls-fork mini-dtls-pthread mini-key-material x509cert-invalid \
+	 mini-dtls-fork dtls-pthread mini-key-material x509cert-invalid \
 	 tls-ext-register tls-supplemental mini-dtls0-9 duplicate-extensions \
 	 record-retvals mini-server-name tls-etm tls-force-etm x509-cert-callback alerts \
 	 client-sign-md5-rep tls12-invalid-key-exchanges session-rdn-read \
@@ -245,8 +245,9 @@ ip_utils_CPPFLAGS = $(AM_CPPFLAGS) \
 
 endif
 
+tls_pthread_LDADD = $(LDADD) -lpthread
 fips_mode_pthread_LDADD = $(LDADD) -lpthread
-mini_dtls_pthread_LDADD = $(LDADD) -lpthread
+dtls_pthread_LDADD = $(LDADD) -lpthread
 rng_pthread_LDADD = $(LDADD) -lpthread
 
 memset0_CFLAGS = -DCHAR=0x0
diff --git a/tests/mini-dtls-pthread.c b/tests/dtls-pthread.c
index 77012553bc..31c2ce5ee4 100644
--- a/tests/mini-dtls-pthread.c
+++ b/tests/dtls-pthread.c
@@ -40,7 +40,9 @@
 # include <sys/wait.h>
 # include <pthread.h>
 #endif
+#include <assert.h>
 #include "utils.h"
+#include "cert-common.h"
 
 #ifdef _WIN32
 
@@ -54,15 +56,6 @@ void doit(void)
 /* These are global */
 pid_t child;
 
-static void terminate(void)
-{
-	int status;
-
-	kill(child, SIGTERM);
-	wait(&status);
-	exit(1);
-}
-
 /* Tests whether we can send and receive from different threads
  * using DTLS, either as server or client. DTLS is a superset of
  * TLS, so correct behavior under fork means TLS would operate too.
@@ -75,76 +68,52 @@ static void tls_log_func(int level, const char *str)
 	fprintf(stderr, "%s|<%d>| %s", side, level, str);
 }
 
-static unsigned char server_cert_pem[] =
-  "-----BEGIN CERTIFICATE-----\n"
-  "MIICHzCCAaWgAwIBAgIBCTAKBggqhkjOPQQDAjA+MQswCQYDVQQGEwJOTDERMA8G\n"
-  "A1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0EwHhcN\n"
-  "MTMwOTI0MTU1MjA0WhcNMjMwOTIyMTU1MjA0WjA0MQswCQYDVQQGEwJOTDERMA8G\n"
-  "A1UEChMIUG9sYXJTU0wxEjAQBgNVBAMTCWxvY2FsaG9zdDBZMBMGByqGSM49AgEG\n"
-  "CCqGSM49AwEHA0IABDfMVtl2CR5acj7HWS3/IG7ufPkGkXTQrRS192giWWKSTuUA\n"
-  "2CMR/+ov0jRdXRa9iojCa3cNVc2KKg76Aci07f+jgZ0wgZowCQYDVR0TBAIwADAd\n"
-  "BgNVHQ4EFgQUUGGlj9QH2deCAQzlZX+MY0anE74wbgYDVR0jBGcwZYAUnW0gJEkB\n"
-  "PyvLeLUZvH4kydv7NnyhQqRAMD4xCzAJBgNVBAYTAk5MMREwDwYDVQQKEwhQb2xh\n"
-  "clNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBFQyBDQYIJAMFD4n5iQ8zoMAoG\n"
-  "CCqGSM49BAMCA2gAMGUCMQCaLFzXptui5WQN8LlO3ddh1hMxx6tzgLvT03MTVK2S\n"
-  "C12r0Lz3ri/moSEpNZWqPjkCMCE2f53GXcYLqyfyJR078c/xNSUU5+Xxl7VZ414V\n"
-  "fGa5kHvHARBPc8YAIVIqDvHH1Q==\n"
-  "-----END CERTIFICATE-----\n";
-
-const gnutls_datum_t server_cert = { server_cert_pem,
-	sizeof(server_cert_pem)
-};
-
-static unsigned char server_key_pem[] =
-  "-----BEGIN EC PRIVATE KEY-----\n"
-  "MHcCAQEEIPEqEyB2AnCoPL/9U/YDHvdqXYbIogTywwyp6/UfDw6noAoGCCqGSM49\n"
-  "AwEHoUQDQgAEN8xW2XYJHlpyPsdZLf8gbu58+QaRdNCtFLX3aCJZYpJO5QDYIxH/\n"
-  "6i/SNF1dFr2KiMJrdw1VzYoqDvoByLTt/w==\n"
-  "-----END EC PRIVATE KEY-----\n";
-
-const gnutls_datum_t server_key = { server_key_pem,
-	sizeof(server_key_pem)
-};
-
 #define MSG "hello1111"
 #define MSG2 "xxxxxxxxxxxx"
 
-static void *start_thread(void *arg)
+#define NO_MSGS 128
+
+static void *recv_thread(void *arg)
 {
 	gnutls_session_t session = arg;
 	int ret;
+	unsigned i;
 	char buf[64];
 
 	if (debug)
 		success("client: TLS version is: %s\n",
 			gnutls_protocol_get_name
 			(gnutls_protocol_get_version(session)));
-	sec_sleep(1);
-	/* the server should reflect our messages */
-	ret = gnutls_record_recv(session, buf, sizeof(buf));
-	if (ret != sizeof(MSG)-1 || memcmp(buf, MSG, sizeof(MSG)-1) != 0) {
-		fail("client: recv failed: %s\n", gnutls_strerror(ret));
-		exit(1);
-	}
 
-	if (debug) {
-		fprintf(stderr, "client received: %.*s\n", ret, buf);
+	for (i=0;i<NO_MSGS;i++) {
+		/* the peer should reflect our messages */
+		do {
+			ret = gnutls_record_recv(session, buf, sizeof(buf));
+		} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+		if (ret < 0)
+			fail("client: recv failed: %s\n", gnutls_strerror(ret));
+		if (ret != sizeof(MSG)-1 || memcmp(buf, MSG, sizeof(MSG)-1) != 0) {
+			fail("client: recv failed; not the expected values (got: %d, exp: %d)\n", ret, (int)sizeof(MSG)-1);
+		}
+
+		if (debug)
+			success("%d: client received: %.*s\n", i, ret, buf);
 	}
 
-	ret = gnutls_record_recv(session, buf, sizeof(buf));
-	if (ret != sizeof(MSG2)-1 || memcmp(buf, MSG2, sizeof(MSG2)-1) != 0) {
+	/* final MSG is MSG2 */
+	do {
+		ret = gnutls_record_recv(session, buf, sizeof(buf));
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+	if (ret < 0)
 		fail("client: recv2 failed: %s\n", gnutls_strerror(ret));
-		exit(1);
-	}
 
-	if (debug) {
-		fprintf(stderr, "client received: %.*s\n", ret, buf);
+	if (ret != sizeof(MSG2)-1 || memcmp(buf, MSG2, sizeof(MSG2)-1) != 0) {
+		fail("client: recv2 failed; not the expected values\n");
 	}
 
-	ret = gnutls_record_recv(session, buf, sizeof(buf));
-	if (ret != 0) {
-		fail("client: recv3 failed: %s\n", gnutls_strerror(ret));
-		exit(1);
+	if (debug) {
+		success("client received: %.*s\n", ret, buf);
+		success("closing recv thread\n");
 	}
 
 	pthread_exit(0);
@@ -154,27 +123,41 @@ static
 void do_thread_stuff(gnutls_session_t session)
 {
 	int ret;
+	unsigned i;
 	pthread_t id;
+	void *rval;
 
+	sec_sleep(1);
 	/* separate sending from receiving */
-	ret = pthread_create(&id, NULL, start_thread, session);
+	ret = pthread_create(&id, NULL, recv_thread, session);
 	if (ret != 0) {
-		exit(1);
+		fail("error in pthread_create\n");
 	}
 
-	ret = gnutls_record_send(session, MSG, sizeof(MSG)-1);
-	if (ret != sizeof(MSG)-1) {
-		fail("client: send failed: %s\n", gnutls_strerror(ret));
-		exit(1);
+	for (i=0;i<NO_MSGS;i++) {
+		do {
+			ret = gnutls_record_send(session, MSG, sizeof(MSG)-1);
+		} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+		if (ret != sizeof(MSG)-1) {
+			fail("client: send failed: %s\n", gnutls_strerror(ret));
+		}
 	}
 
-	ret = gnutls_record_send(session, MSG2, sizeof(MSG2)-1);
+	do {
+		ret = gnutls_record_send(session, MSG2, sizeof(MSG2)-1);
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
 	if (ret != sizeof(MSG2)-1) {
 		fail("client: send2 failed: %s\n", gnutls_strerror(ret));
-		exit(1);
 	}
-	sec_sleep(2);
-	gnutls_bye(session, GNUTLS_SHUT_WR);
+
+	if (debug)
+		success("closing sending thread\n");
+	assert(pthread_join(id, &rval)==0);
+	assert(rval == 0);
+
+	do {
+		ret = gnutls_bye(session, GNUTLS_SHUT_RDWR);
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
 }
 
 static void do_reflect_stuff(gnutls_session_t session)
@@ -184,38 +167,43 @@ static void do_reflect_stuff(gnutls_session_t session)
 	int ret;
 
 	do {
-		ret = gnutls_record_recv(session, buf, sizeof(buf));
+		do {
+			ret = gnutls_record_recv(session, buf, sizeof(buf));
+		} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
 		if (ret < 0) {
 			fail("server: recv failed: %s\n", gnutls_strerror(ret));
-			terminate();
 		}
 
-		if (ret == 0)
+		if (ret == 0) {
 			break;
+		}
 
 		buf_size = ret;
 		if (debug) {
-			fprintf(stderr, "server received: %.*s\n", buf_size, buf);
+			success("server received: %.*s\n", buf_size, buf);
 		}
 
-		ret = gnutls_record_send(session, buf, buf_size);
+		do {
+			ret = gnutls_record_send(session, buf, buf_size);
+		} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
 		if (ret < 0) {
 			fail("server: send failed: %s\n", gnutls_strerror(ret));
-			terminate();
 		}
+		if (debug)
+			success("reflected %d\n", ret);
 	} while(1);
 
-	/* do not wait for the peer to close the connection.
-	 */
-	gnutls_bye(session, GNUTLS_SHUT_WR);
+	do {
+		gnutls_bye(session, GNUTLS_SHUT_WR);
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
 }
 
-static void client(int fd, unsigned do_thread)
+static void client(int fd, const char *prio, unsigned do_thread, unsigned false_start)
 {
 	int ret;
 	gnutls_certificate_credentials_t x509_cred;
 	gnutls_session_t session;
-	/* Need to enable anonymous KX specifically. */
+	unsigned flags = GNUTLS_CLIENT;
 
 	global_init();
 
@@ -225,37 +213,28 @@ static void client(int fd, unsigned do_thread)
 		gnutls_global_set_log_level(4711);
 	}
 
-	gnutls_certificate_allocate_credentials(&x509_cred);
+	assert(gnutls_certificate_allocate_credentials(&x509_cred)>=0);
 
-	/* Initialize TLS session
-	 */
-	gnutls_init(&session, GNUTLS_CLIENT | GNUTLS_DATAGRAM);
+	if (false_start)
+		flags |= GNUTLS_ENABLE_FALSE_START;
+
+	assert(gnutls_init(&session, flags|GNUTLS_DATAGRAM) >= 0);
 	gnutls_dtls_set_mtu(session, 1500);
 	gnutls_dtls_set_timeouts(session, 6 * 1000, 60 * 1000);
-	//gnutls_transport_set_push_function(session, push);
 
-	/* Use default priorities */
-	gnutls_priority_set_direct(session,
-				   "NONE:+VERS-DTLS-ALL:+CIPHER-ALL:+MAC-ALL:+SIGN-ALL:+COMP-ALL:+ECDHE-ECDSA:+CURVE-ALL",
-				   NULL);
+	assert(gnutls_priority_set_direct(session, prio, NULL)>=0);
 
-	/* put the anonymous credentials to the current session
-	 */
-	gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, x509_cred);
+	assert(gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, x509_cred)>=0);
 
 	gnutls_transport_set_int(session, fd);
 
-	/* Perform the TLS handshake
-	 */
 	do {
 		ret = gnutls_handshake(session);
 	}
 	while (ret < 0 && gnutls_error_is_fatal(ret) == 0);
 
 	if (ret < 0) {
-		fail("client: Handshake failed\n");
-		gnutls_perror(ret);
-		exit(1);
+		fail("client: Handshake failed: %s\n", gnutls_strerror(ret));
 	} else {
 		if (debug)
 			success("client: Handshake was completed\n");
@@ -273,18 +252,14 @@ static void client(int fd, unsigned do_thread)
 	gnutls_certificate_free_credentials(x509_cred);
 
 	gnutls_global_deinit();
-	exit(0);
 }
 
-
-static void server(int fd, unsigned do_thread)
+static void server(int fd, const char *prio, unsigned do_thread)
 {
 	int ret;
 	gnutls_certificate_credentials_t x509_cred;
 	gnutls_session_t session;
 
-	/* this must be called once in the program
-	 */
 	global_init();
 
 #if 0
@@ -295,21 +270,16 @@ static void server(int fd, unsigned do_thread)
 	}
 #endif
 
-	gnutls_certificate_allocate_credentials(&x509_cred);
-	gnutls_certificate_set_x509_key_mem(x509_cred, &server_cert,
+	assert(gnutls_certificate_allocate_credentials(&x509_cred)>=0);
+	assert(gnutls_certificate_set_x509_key_mem(x509_cred, &server_cert,
 					    &server_key,
-					    GNUTLS_X509_FMT_PEM);
+					    GNUTLS_X509_FMT_PEM)>=0);
 
-	gnutls_init(&session, GNUTLS_SERVER | GNUTLS_DATAGRAM);
+	assert(gnutls_init(&session, GNUTLS_SERVER | GNUTLS_DATAGRAM)>=0);
 	gnutls_dtls_set_timeouts(session, 5 * 1000, 60 * 1000);
 	gnutls_dtls_set_mtu(session, 400);
 
-	/* avoid calling all the priority functions, since the defaults
-	 * are adequate.
-	 */
-	gnutls_priority_set_direct(session,
-				   "NONE:+VERS-DTLS1.2:+CIPHER-ALL:+MAC-ALL:+SIGN-ALL:+COMP-ALL:+ECDHE-ECDSA:+CURVE-ALL",
-				   NULL);
+	assert(gnutls_priority_set_direct(session, prio, NULL)>=0);
 
 	gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, x509_cred);
 
@@ -323,7 +293,6 @@ static void server(int fd, unsigned do_thread)
 		gnutls_deinit(session);
 		fail("server: Handshake has failed (%s)\n\n",
 		     gnutls_strerror(ret));
-		terminate();
 	}
 	if (debug)
 		success("server: Handshake was completed\n");
@@ -351,11 +320,14 @@ static void server(int fd, unsigned do_thread)
 }
 
 static
-void run(unsigned do_thread)
+void run(const char *str, const char *prio, unsigned do_thread, unsigned false_start)
 {
 	int fd[2];
 	int ret;
 
+	if (str)
+		success("running %s\n", str);
+
 	ret = socketpair(AF_UNIX, SOCK_STREAM, 0, fd);
 	if (ret < 0) {
 		perror("socketpair");
@@ -374,12 +346,12 @@ void run(unsigned do_thread)
 		/* parent */
 
 		close(fd[1]);
-		client(fd[0], do_thread);
+		client(fd[0], prio, do_thread, false_start);
 		wait(&status);
 		check_wait_status(status);
 	} else {
 		close(fd[0]);
-		server(fd[1], 1-do_thread);
+		server(fd[1], prio, 1-do_thread);
 		exit(0);
 	}
 }
@@ -387,7 +359,11 @@ void run(unsigned do_thread)
 void doit(void)
 {
 	signal(SIGPIPE, SIG_IGN);
-	run(0);
-	run(1);
+	run("default, threaded client", "NORMAL", 0, 0);
+	run("default, threaded server", "NORMAL", 1, 0);
+	run("dtls1.2, threaded client", "NORMAL:-VERS-ALL:+VERS-DTLS1.2", 0, 0);
+	run("dtls1.2, threaded server", "NORMAL:-VERS-ALL:+VERS-DTLS1.2", 1, 0);
+	run("dtls1.2 false start, threaded client", "NORMAL:-VERS-ALL:+VERS-DTLS1.2", 0, 1);
+	run("dtls1.2 false start, threaded server", "NORMAL:-VERS-ALL:+VERS-DTLS1.2", 1, 1);
 }
 #endif				/* _WIN32 */
diff --git a/tests/tls-pthread.c b/tests/tls-pthread.c
new file mode 100644
index 0000000000..0d8eb27a51
--- /dev/null
+++ b/tests/tls-pthread.c
@@ -0,0 +1,388 @@
+/*
+ * Copyright (C) 2019 Nikos Mavrogiannopoulos
+ *
+ * Author: Nikos Mavrogiannopoulos
+ *
+ * This file is part of GnuTLS.
+ *
+ * GnuTLS is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GnuTLS is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with GnuTLS; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <gnutls/gnutls.h>
+#include <gnutls/dtls.h>
+#include <signal.h>
+#include <unistd.h>
+#ifndef _WIN32
+# include <netinet/in.h>
+# include <sys/types.h>
+# include <sys/socket.h>
+# include <sys/wait.h>
+# include <pthread.h>
+#endif
+#include <assert.h>
+#include "utils.h"
+#include "cert-common.h"
+
+#define FLAG_CLI_DO_THREADS 1
+#define FLAG_FALSE_START 2
+#define FLAG_EARLY_START 4
+
+#ifdef _WIN32
+
+void doit(void)
+{
+	exit(77);
+}
+
+#else
+
+/* Tests whether we can send and receive from different threads
+ * using DTLS, either as server or client. DTLS is a superset of
+ * TLS, so correct behavior under fork means TLS would operate too.
+ */
+
+const char *side = "";
+
+static void tls_log_func(int level, const char *str)
+{
+	fprintf(stderr, "%s|<%d>| %s", side, level, str);
+}
+
+#define MSG "hello1111"
+#define MSG2 "xxxxxxxxxxxx"
+
+#define NO_MSGS 128
+
+static void *recv_thread(void *arg)
+{
+	gnutls_session_t session = arg;
+	int ret;
+	unsigned i;
+	char buf[64];
+
+	if (debug)
+		success("client: TLS version is: %s\n",
+			gnutls_protocol_get_name
+			(gnutls_protocol_get_version(session)));
+
+	for (i=0;i<NO_MSGS;i++) {
+		/* the peer should reflect our messages */
+		do {
+			ret = gnutls_record_recv(session, buf, sizeof(buf));
+		} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+		if (ret < 0)
+			fail("client: recv failed: %s\n", gnutls_strerror(ret));
+		if (ret != sizeof(MSG)-1 || memcmp(buf, MSG, sizeof(MSG)-1) != 0) {
+			fail("client: recv failed; not the expected values (got: %d, exp: %d)\n", ret, (int)sizeof(MSG)-1);
+		}
+
+		if (debug)
+			success("%d: client received: %.*s\n", i, ret, buf);
+	}
+
+	/* final MSG is MSG2 */
+	do {
+		ret = gnutls_record_recv(session, buf, sizeof(buf));
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+	if (ret < 0)
+		fail("client: recv2 failed: %s\n", gnutls_strerror(ret));
+
+	if (ret != sizeof(MSG2)-1 || memcmp(buf, MSG2, sizeof(MSG2)-1) != 0) {
+		fail("client: recv2 failed; not the expected values\n");
+	}
+
+	do {
+		ret = gnutls_record_recv(session, buf, sizeof(buf));
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+	if (ret != 0)
+		fail("client: no EOF: %d\n", ret);
+
+	if (debug) {
+		success("client received: %.*s\n", ret, buf);
+		success("closing recv thread\n");
+	}
+
+	pthread_exit(0);
+}
+
+static
+void do_thread_stuff(gnutls_session_t session)
+{
+	int ret;
+	unsigned i;
+	pthread_t id;
+	void *rval;
+
+	sec_sleep(1);
+	/* separate sending from receiving */
+	ret = pthread_create(&id, NULL, recv_thread, session);
+	if (ret != 0) {
+		fail("error in pthread_create\n");
+	}
+
+	for (i=0;i<NO_MSGS;i++) {
+		do {
+			ret = gnutls_record_send(session, MSG, sizeof(MSG)-1);
+		} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+		if (ret != sizeof(MSG)-1) {
+			fail("client: send failed: %s\n", gnutls_strerror(ret));
+		}
+	}
+
+	do {
+		ret = gnutls_record_send(session, MSG2, sizeof(MSG2)-1);
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+	if (ret != sizeof(MSG2)-1) {
+		fail("client: send2 failed: %s\n", gnutls_strerror(ret));
+	}
+
+	if (debug)
+		success("closing sending thread\n");
+
+	do {
+		ret = gnutls_bye(session, GNUTLS_SHUT_WR);
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+
+	/* the receiving thread will receive the EOF and close */
+
+	assert(pthread_join(id, &rval)==0);
+	assert(rval == 0);
+
+}
+
+static void do_reflect_stuff(gnutls_session_t session)
+{
+	char buf[64];
+	unsigned buf_size;
+	int ret;
+
+	do {
+		do {
+			ret = gnutls_record_recv(session, buf, sizeof(buf));
+		} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+		if (ret < 0) {
+			fail("server: recv failed: %s\n", gnutls_strerror(ret));
+		}
+
+		if (ret == 0) {
+			break;
+		}
+
+		buf_size = ret;
+		if (debug) {
+			success("server received: %.*s\n", buf_size, buf);
+		}
+
+		do {
+			ret = gnutls_record_send(session, buf, buf_size);
+		} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+		if (ret < 0) {
+			fail("server: send failed: %s\n", gnutls_strerror(ret));
+		}
+		if (debug)
+			success("reflected %d\n", ret);
+	} while(1);
+
+	do {
+		gnutls_bye(session, GNUTLS_SHUT_WR);
+	} while(ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED);
+}
+
+static void client(int fd, const char *prio, unsigned flags)
+{
+	int ret;
+	gnutls_certificate_credentials_t x509_cred;
+	gnutls_session_t session;
+	unsigned init_flags = GNUTLS_CLIENT;
+
+	global_init();
+
+	if (debug) {
+		side = "client";
+		gnutls_global_set_log_function(tls_log_func);
+		gnutls_global_set_log_level(4711);
+	}
+
+	assert(gnutls_certificate_allocate_credentials(&x509_cred)>=0);
+
+	if (flags & FLAG_FALSE_START)
+		init_flags |= GNUTLS_ENABLE_FALSE_START;
+
+	assert(gnutls_init(&session, init_flags) >= 0);
+	gnutls_handshake_set_timeout(session, 60 * 1000);
+
+	assert(gnutls_priority_set_direct(session, prio, NULL)>=0);
+
+	assert(gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, x509_cred)>=0);
+
+	gnutls_transport_set_int(session, fd);
+
+	/* Perform the TLS handshake
+	 */
+	do {
+		ret = gnutls_handshake(session);
+	}
+	while (ret < 0 && gnutls_error_is_fatal(ret) == 0);
+
+	if (ret < 0) {
+		fail("client: Handshake failed: %s\n", gnutls_strerror(ret));
+	} else {
+		if (debug)
+			success("client: Handshake was completed\n");
+	}
+
+	if (flags & FLAG_CLI_DO_THREADS)
+		do_thread_stuff(session);
+	else
+		do_reflect_stuff(session);
+
+	close(fd);
+
+	gnutls_deinit(session);
+
+	gnutls_certificate_free_credentials(x509_cred);
+
+	gnutls_global_deinit();
+}
+
+static void server(int fd, const char *prio, unsigned flags)
+{
+	int ret;
+	gnutls_certificate_credentials_t x509_cred;
+	gnutls_session_t session;
+	unsigned init_flags = GNUTLS_SERVER;
+
+	/* this must be called once in the program
+	 */
+	global_init();
+
+#if 0
+	if (debug) {
+		side = "server";
+		gnutls_global_set_log_function(tls_log_func);
+		gnutls_global_set_log_level(4711);
+	}
+#endif
+
+	assert(gnutls_certificate_allocate_credentials(&x509_cred)>=0);
+	assert(gnutls_certificate_set_x509_key_mem(x509_cred, &server_cert,
+					    &server_key,
+					    GNUTLS_X509_FMT_PEM)>=0);
+
+	if (flags & FLAG_EARLY_START)
+		init_flags |= GNUTLS_ENABLE_EARLY_START;
+
+	assert(gnutls_init(&session, init_flags)>=0);
+	gnutls_handshake_set_timeout(session, 60 * 1000);
+
+	assert(gnutls_priority_set_direct(session, prio, NULL)>=0);
+
+	gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, x509_cred);
+
+	gnutls_transport_set_int(session, fd);
+
+	do {
+		ret = gnutls_handshake(session);
+	} while (ret < 0 && gnutls_error_is_fatal(ret) == 0);
+	if (ret < 0) {
+		close(fd);
+		gnutls_deinit(session);
+		fail("server: Handshake has failed (%s)\n\n",
+		     gnutls_strerror(ret));
+	}
+	if (debug)
+		success("server: Handshake was completed\n");
+
+	if (debug)
+		success("server: TLS version is: %s\n",
+			gnutls_protocol_get_name
+			(gnutls_protocol_get_version(session)));
+
+	if (flags & FLAG_CLI_DO_THREADS)
+		do_reflect_stuff(session);
+	else
+		do_thread_stuff(session);
+
+
+	close(fd);
+	gnutls_deinit(session);
+
+	gnutls_certificate_free_credentials(x509_cred);
+
+	gnutls_global_deinit();
+
+	if (debug)
+		success("server: finished\n");
+}
+
+static
+void run(const char *str, const char *prio, unsigned flags)
+{
+	int fd[2];
+	int ret;
+	pid_t child;
+
+	if (str)
+		success("running %s\n", str);
+
+	ret = socketpair(AF_UNIX, SOCK_STREAM, 0, fd);
+	if (ret < 0) {
+		perror("socketpair");
+		exit(1);
+	}
+
+	child = fork();
+	if (child < 0) {
+		perror("fork");
+		fail("fork");
+	}
+
+	if (child) {
+		int status;
+		/* parent */
+
+		close(fd[1]);
+		client(fd[0], prio, flags);
+		wait(&status);
+		check_wait_status(status);
+	} else {
+		close(fd[0]);
+		server(fd[1], prio, flags);
+		exit(0);
+	}
+}
+
+void doit(void)
+{
+	signal(SIGPIPE, SIG_IGN);
+	run("default, threaded client", "NORMAL", FLAG_CLI_DO_THREADS);
+	run("default, threaded server", "NORMAL", 0);
+	run("tls1.2, threaded client", "NORMAL:-VERS-TLS-ALL:+VERS-TLS1.2", FLAG_CLI_DO_THREADS);
+	run("tls1.2, threaded server", "NORMAL:-VERS-TLS-ALL:+VERS-TLS1.2", 0);
+	run("tls1.2 false start, threaded client", "NORMAL:-VERS-TLS-ALL:+VERS-TLS1.2", FLAG_CLI_DO_THREADS|FLAG_FALSE_START);
+	run("tls1.2 false start, threaded server", "NORMAL:-VERS-TLS-ALL:+VERS-TLS1.2", FLAG_FALSE_START);
+	run("tls1.3 early start, threaded client", "NORMAL:-VERS-TLS-ALL:+VERS-TLS1.3", FLAG_CLI_DO_THREADS|FLAG_EARLY_START);
+	run("tls1.3 early start, threaded server", "NORMAL:-VERS-TLS-ALL:+VERS-TLS1.3", FLAG_EARLY_START);
+}
+#endif				/* _WIN32 */