Initial Async notify code changes

Initial API implemented for notifying applications that an ASYNC_JOB
has completed. Currently only s_server is using this. The Dummy Async
engine "cheats" in that it notifies that it has completed *before* it
pauses the job. A normal async engine would not do that.

Only the posix version of this has been implemented so far, so it will
probably fail to compile on Windows at the moment.

Reviewed-by: Rich Salz <rsalz@openssl.org>

9 files changed, 140 insertions, 12 deletions

diff --git a/apps/s_server.c b/apps/s_server.c
index 6fb8f675a3..75a3ba0a36 100644
--- a/apps/s_server.c
+++ b/apps/s_server.c
@@ -194,6 +194,7 @@ typedef unsigned int u_int;
 static RSA *tmp_rsa_cb(SSL *s, int is_export, int keylength);
 #endif
 static int not_resumable_sess_cb(SSL *s, int is_forward_secure);
+static void wait_for_async(SSL *s);
 static int sv_body(char *hostname, int s, int stype, unsigned char *context);
 static int www_body(char *hostname, int s, int stype, unsigned char *context);
 static int rev_body(char *hostname, int s, int stype, unsigned char *context);
@@ -1735,7 +1736,7 @@ int s_server_main(int argc, char *argv[])
             SSL_CTX_sess_set_cache_size(ctx2, 128);
 
         if (async)
-			SSL_CTX_set_mode(ctx2, SSL_MODE_ASYNC);
+            SSL_CTX_set_mode(ctx2, SSL_MODE_ASYNC);
 
         if ((!SSL_CTX_load_verify_locations(ctx2, CAfile, CApath)) ||
             (!SSL_CTX_set_default_verify_paths(ctx2))) {
@@ -2007,6 +2008,21 @@ static void print_stats(BIO *bio, SSL_CTX *ssl_ctx)
                SSL_CTX_sess_get_cache_size(ssl_ctx));
 }
 
+static void wait_for_async(SSL *s)
+{
+    int width, fd;
+    fd_set asyncfds;
+
+    fd = SSL_get_async_wait_fd(s);
+    if (!fd)
+        return;
+
+    width = fd + 1;
+    FD_ZERO(&asyncfds);
+    openssl_fdset(fd, &asyncfds);
+    select(width, (void *)&asyncfds, NULL, NULL, NULL);
+}
+
 static int sv_body(char *hostname, int s, int stype, unsigned char *context)
 {
     char *buf = NULL;
@@ -2302,6 +2318,7 @@ static int sv_body(char *hostname, int s, int stype, unsigned char *context)
                     break;
                 case SSL_ERROR_WANT_ASYNC:
                     BIO_printf(bio_s_out, "Write BLOCK (Async)\n");
+                    wait_for_async(con);
                     break;
                 case SSL_ERROR_WANT_WRITE:
                 case SSL_ERROR_WANT_READ:
@@ -2368,6 +2385,9 @@ static int sv_body(char *hostname, int s, int stype, unsigned char *context)
                         goto again;
                     break;
                 case SSL_ERROR_WANT_ASYNC:
+                    BIO_printf(bio_s_out, "Read BLOCK (Async)\n");
+                    wait_for_async(con);
+                    break;
                 case SSL_ERROR_WANT_WRITE:
                 case SSL_ERROR_WANT_READ:
                     BIO_printf(bio_s_out, "Read BLOCK\n");
diff --git a/crypto/async/arch/async_posix.c b/crypto/async/arch/async_posix.c
index bb799e54f8..8474ea4ba1 100644
--- a/crypto/async/arch/async_posix.c
+++ b/crypto/async/arch/async_posix.c
@@ -57,6 +57,7 @@
 #ifdef ASYNC_SYSV
 # include <stddef.h>
 # include <ucontext.h>
+# include <unistd.h>
 # include <openssl/crypto.h>
 # include <openssl/async.h>
 
@@ -85,4 +86,29 @@ void ASYNC_FIBRE_free(ASYNC_FIBRE *fibre)
     if (fibre->fibre.uc_stack.ss_sp)
         OPENSSL_free(fibre->fibre.uc_stack.ss_sp);
 }
+
+int async_pipe(int *pipefds)
+{
+    if (pipe(pipefds) == 0)
+        return 1;
+
+    return 0;
+}
+
+int async_write1(int fd, const void *buf)
+{
+    if (write(fd, buf, 1) > 0)
+        return 1;
+
+    return 0;
+}
+
+int async_read1(int fd, void *buf)
+{
+    if (read(fd, buf, 1) > 0)
+        return 1;
+
+    return 0;
+}
+
 #endif
diff --git a/crypto/async/arch/async_posix.h b/crypto/async/arch/async_posix.h
index 43cea6d9b0..7b8e905bec 100644
--- a/crypto/async/arch/async_posix.h
+++ b/crypto/async/arch/async_posix.h
@@ -99,5 +99,10 @@ static inline int ASYNC_FIBRE_swapcontext(ASYNC_FIBRE *o, ASYNC_FIBRE *n, int r)
 int ASYNC_FIBRE_init(ASYNC_FIBRE *fibre);
 void ASYNC_FIBRE_free(ASYNC_FIBRE *fibre);
 
+int async_pipe(int *pipefds);
+int async_write1(int fd, const void *buf);
+int async_read1(int fd, void *buf);
+
+
 # endif
 #endif
diff --git a/crypto/async/async.c b/crypto/async/async.c
index 6fed1979f9..bf026ee9aa 100644
--- a/crypto/async/async.c
+++ b/crypto/async/async.c
@@ -106,11 +106,21 @@ static int ASYNC_CTX_free(void)
 static ASYNC_JOB *ASYNC_JOB_new(void)
 {
     ASYNC_JOB *job = NULL;
+    int pipefds[2];
 
     if(!(job = OPENSSL_malloc(sizeof (ASYNC_JOB)))) {
         return NULL;
     }
 
+    if(!async_pipe(pipefds)) {
+        OPENSSL_free(job);
+        return NULL;
+    }
+
+    job->wake_set = 0;
+    job->wait_fd = pipefds[0];
+    job->wake_fd = pipefds[1];
+
     job->status = ASYNC_JOB_RUNNING;
     job->funcargs = NULL;
 
@@ -335,3 +345,36 @@ void ASYNC_free_pool(void)
     } while (job);
     sk_ASYNC_JOB_free(pool);
 }
+
+ASYNC_JOB *ASYNC_get_current_job(void)
+{
+    ASYNC_CTX *ctx;
+    if((ctx = ASYNC_get_ctx()) == NULL)
+        return NULL;
+
+    return ctx->currjob;
+}
+
+int ASYNC_get_wait_fd(ASYNC_JOB *job)
+{
+    return job->wait_fd;
+}
+
+void ASYNC_wake(ASYNC_JOB *job)
+{
+    char dummy = 0;
+
+    if (job->wake_set)
+        return;
+    async_write1(job->wake_fd, &dummy);
+    job->wake_set = 1;
+}
+
+void ASYNC_clear_wake(ASYNC_JOB *job)
+{
+    char dummy = 0;
+    if (!job->wake_set)
+        return;
+    async_read1(job->wait_fd, &dummy);
+    job->wake_set = 0;
+}
diff --git a/crypto/async/async_locl.h b/crypto/async/async_locl.h
index 4a6cb3e2c5..56480c8b0e 100644
--- a/crypto/async/async_locl.h
+++ b/crypto/async/async_locl.h
@@ -70,6 +70,9 @@ struct async_job_st {
     void *funcargs;
     int ret;
     int status;
+    int wake_set;
+    int wait_fd;
+    int wake_fd;
 };
 
 void ASYNC_start_func(void);
diff --git a/engines/e_dasync.c b/engines/e_dasync.c
index 6c41f7522d..9e0ed1b492 100644
--- a/engines/e_dasync.c
+++ b/engines/e_dasync.c
@@ -82,6 +82,7 @@ static int dasync_digests(ENGINE *e, const EVP_MD **digest,
 
 static int dasync_digest_nids[] = { NID_sha1, 0 };
 
+static void dummy_pause_job(void);
 
 /* SHA1 */
 static int digest_sha1_init(EVP_MD_CTX *ctx);
@@ -234,6 +235,25 @@ static int dasync_digests(ENGINE *e, const EVP_MD **digest,
     return ok;
 }
 
+static void dummy_pause_job(void) {
+    ASYNC_JOB *job;
+
+    if ((job = ASYNC_get_current_job()) == NULL)
+        return;
+
+    /*
+     * In the Dummy async engine we are cheating. We signal that the job
+     * is complete by waking it before the call to ASYNC_pause_job(). A real
+     * async engine would only wake when the job was actually complete
+     */
+    ASYNC_wake(job);
+
+    /* Ignore errors - we carry on anyway */
+    ASYNC_pause_job();
+
+    ASYNC_clear_wake(job);
+}
+
 
 /*
  * SHA1 implementation. At the moment we just defer to the standard
@@ -243,8 +263,7 @@ static int dasync_digests(ENGINE *e, const EVP_MD **digest,
 #define data(ctx) ((SHA_CTX *)(ctx)->md_data)
 static int digest_sha1_init(EVP_MD_CTX *ctx)
 {
-    /* Ignore errors - we carry on anyway */
-    ASYNC_pause_job();
+    dummy_pause_job();
 
     return SHA1_Init(data(ctx));
 }
@@ -252,16 +271,14 @@ static int digest_sha1_init(EVP_MD_CTX *ctx)
 static int digest_sha1_update(EVP_MD_CTX *ctx, const void *data,
                              unsigned long count)
 {
-    /* Ignore errors - we carry on anyway */
-    ASYNC_pause_job();
+    dummy_pause_job();
 
     return SHA1_Update(data(ctx), data, (size_t)count);
 }
 
 static int digest_sha1_final(EVP_MD_CTX *ctx, unsigned char *md)
 {
-    /* Ignore errors - we carry on anyway */
-    ASYNC_pause_job();
+    dummy_pause_job();
 
     return SHA1_Final(md, data(ctx));
 }
@@ -273,14 +290,14 @@ static int digest_sha1_final(EVP_MD_CTX *ctx, unsigned char *md)
 static int dasync_pub_enc(int flen, const unsigned char *from,
                     unsigned char *to, RSA *rsa, int padding) {
     /* Ignore errors - we carry on anyway */
-    ASYNC_pause_job();
+    dummy_pause_job();
     return RSA_PKCS1_OpenSSL()->rsa_pub_enc(flen, from, to, rsa, padding);
 }
 
 static int dasync_pub_dec(int flen, const unsigned char *from,
                     unsigned char *to, RSA *rsa, int padding) {
     /* Ignore errors - we carry on anyway */
-    ASYNC_pause_job();
+    dummy_pause_job();
     return RSA_PKCS1_OpenSSL()->rsa_pub_dec(flen, from, to, rsa, padding);
 }
 
@@ -288,7 +305,7 @@ static int dasync_rsa_priv_enc(int flen, const unsigned char *from,
                       unsigned char *to, RSA *rsa, int padding)
 {
     /* Ignore errors - we carry on anyway */
-    ASYNC_pause_job();
+    dummy_pause_job();
     return RSA_PKCS1_OpenSSL()->rsa_priv_enc(flen, from, to, rsa, padding);
 }
 
@@ -296,14 +313,14 @@ static int dasync_rsa_priv_dec(int flen, const unsigned char *from,
                       unsigned char *to, RSA *rsa, int padding)
 {
     /* Ignore errors - we carry on anyway */
-    ASYNC_pause_job();
+    dummy_pause_job();
     return RSA_PKCS1_OpenSSL()->rsa_priv_dec(flen, from, to, rsa, padding);
 }
 
 static int dasync_rsa_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx)
 {
     /* Ignore errors - we carry on anyway */
-    ASYNC_pause_job();
+    dummy_pause_job();
     return RSA_PKCS1_OpenSSL()->rsa_mod_exp(r0, I, rsa, ctx);
 }
 
diff --git a/include/openssl/async.h b/include/openssl/async.h
index 1cd799202f..45e2f6e206 100644
--- a/include/openssl/async.h
+++ b/include/openssl/async.h
@@ -75,6 +75,11 @@ int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
 int ASYNC_pause_job(void);
 int ASYNC_in_job(void);
 
+int ASYNC_get_wait_fd(ASYNC_JOB *job);
+ASYNC_JOB *ASYNC_get_current_job(void);
+void ASYNC_wake(ASYNC_JOB *job);
+void ASYNC_clear_wake(ASYNC_JOB *job);
+
 # ifdef  __cplusplus
 }
 # endif
diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h
index eaac180037..ed9bc5bc57 100644
--- a/include/openssl/ssl.h
+++ b/include/openssl/ssl.h
@@ -1572,6 +1572,7 @@ __owur char *SSL_get_srp_userinfo(SSL *s);
 void SSL_certs_clear(SSL *s);
 void SSL_free(SSL *ssl);
 __owur int SSL_waiting_for_async(SSL *s);
+__owur int SSL_get_async_wait_fd(SSL *s);
 __owur int SSL_accept(SSL *ssl);
 __owur int SSL_connect(SSL *ssl);
 __owur int SSL_read(SSL *ssl, void *buf, int num);
diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index 31adbe473e..270a908a3e 100644
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -931,6 +931,14 @@ int SSL_waiting_for_async(SSL *s)
     return 0;
 }
 
+int SSL_get_async_wait_fd(SSL *s)
+{
+    if (!s->job)
+        return 0;
+
+    return ASYNC_get_wait_fd(s->job);
+}
+
 static int ssl_accept_intern(void *vargs)
 {
     struct ssl_async_args *args;