MINOR: sample: Expose SSL captures using new fetchers

To be able to provide JA3 compatible TLS Fingerprints we need to expose
all Client Hello captured data using fetchers. Patch provides new
and modifies existing fetchers to add ability to filter out GREASE values:
- ssl_fc_cipherlist_*
- ssl_fc_ecformats_bin
- ssl_fc_eclist_bin
- ssl_fc_extlist_bin
- ssl_fc_protocol_hello_id

4 files changed, 276 insertions, 21 deletions

diff --git a/doc/configuration.txt b/doc/configuration.txt
index 9a8d97d0e..ba6727a53 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -18875,27 +18875,103 @@ ssl_fc_cipher : string
   Returns the name of the used cipher when the incoming connection was made
   over an SSL/TLS transport layer.
 
-ssl_fc_cipherlist_bin : binary
-  Returns the binary form of the client hello cipher list. The maximum returned
-  value length is according with the value of
-  "tune.ssl.capture-cipherlist-size".
+ssl_fc_cipherlist_bin([<filter_option>]) : binary
+  Returns the binary form of the client hello cipher list. The maximum
+  returned value length is limited by the shared capture buffer size
+  controlled by "tune.ssl.capture-cipherlist-size" setting. Setting
+  <filter_option> allows to filter returned data. Accepted values:
+    0 : return the full list of ciphers (default)
+    1 : exclude GREASE (RFC8701) values from the output
 
-ssl_fc_cipherlist_hex : string
+  Example:
+      http-request set-header X-SSL-JA3 %[ssl_fc_protocol_hello_id],\
+          %[ssl_fc_cipherlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_extlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_eclist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_ecformats_bin,be2dec(-,1)]
+      acl is_malware req.fhdr(x-ssl-ja3),digest(md5),hex \
+          -f /path/to/file/with/malware-ja3.lst
+      http-request set-header X-Malware True if is_malware
+      http-request set-header X-Malware False if !is_malware
+
+ssl_fc_cipherlist_hex([<filter_option>]) : string
   Returns the binary form of the client hello cipher list encoded as
-  hexadecimal. The maximum returned value length is according with the value of
-  "tune.ssl.capture-cipherlist-size".
-
-ssl_fc_cipherlist_str : string
+  hexadecimal. The maximum returned value length is limited by the shared
+  capture buffer size controlled by "tune.ssl.capture-cipherlist-size"
+  setting.  Setting <filter_option> allows to filter returned data. Accepted
+  values:
+    0 : return the full list of ciphers (default)
+    1 : exclude GREASE (RFC8701) values from the output
+
+ssl_fc_cipherlist_str([<filter_option>]) : string
   Returns the decoded text form of the client hello cipher list. The maximum
-  number of ciphers returned is according with the value of
-  "tune.ssl.capture-cipherlist-size". Note that this sample-fetch is only
-  available with OpenSSL >= 1.0.2. If the function is not enabled, this
-  sample-fetch returns the hash like "ssl_fc_cipherlist_xxh".
+  returned value length is limited by the shared capture buffer size
+  controlled by "tune.ssl.capture-cipherlist-size" setting. Setting
+  <filter_option> allows to filter returned data. Accepted values:
+    0 : return the full list of ciphers (default)
+    1 : exclude GREASE (RFC8701) values from the output
+  Note that this sample-fetch is only available with OpenSSL >= 1.0.2. If the
+  function is not enabled, this sample-fetch returns the hash like
+  "ssl_fc_cipherlist_xxh".
 
 ssl_fc_cipherlist_xxh : integer
-  Returns a xxh64 of the cipher list. This hash can be return only is the value
+  Returns a xxh64 of the cipher list. This hash can return only if the value
   "tune.ssl.capture-cipherlist-size" is set greater than 0, however the hash
-  take in account all the data of the cipher list.
+  take into account all the data of the cipher list.
+
+ssl_fc_ecformats_bin : binary
+  Return the binary form of the client hello supported elliptic curve point
+  formats. The maximum returned value length is limited by the shared capture
+  buffer size controlled by "tune.ssl.capture-cipherlist-size" setting.
+
+  Example:
+      http-request set-header X-SSL-JA3 %[ssl_fc_protocol_hello_id],\
+          %[ssl_fc_cipherlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_extlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_eclist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_ecformats_bin,be2dec(-,1)]
+      acl is_malware req.fhdr(x-ssl-ja3),digest(md5),hex \
+          -f /path/to/file/with/malware-ja3.lst
+      http-request set-header X-Malware True if is_malware
+      http-request set-header X-Malware False if !is_malware
+
+ssl_fc_eclist_bin([<filter_option>]) : binary
+  Returns the binary form of the client hello supported elliptic curves. The
+  maximum returned value length is limited by the shared capture buffer size
+  controlled by "tune.ssl.capture-cipherlist-size" setting. Setting
+  <filter_option> allows to filter returned data. Accepted values:
+    0 : return the full list of supported elliptic curves (default)
+    1 : exclude GREASE (RFC8701) values from the output
+
+  Example:
+      http-request set-header X-SSL-JA3 %[ssl_fc_protocol_hello_id],\
+          %[ssl_fc_cipherlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_extlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_eclist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_ecformats_bin,be2dec(-,1)]
+      acl is_malware req.fhdr(x-ssl-ja3),digest(md5),hex \
+          -f /path/to/file/with/malware-ja3.lst
+      http-request set-header X-Malware True if is_malware
+      http-request set-header X-Malware False if !is_malware
+
+ssl_fc_extlist_bin([<filter_option>]) : binary
+  Returns the binary form of the client hello extension list. The maximum
+  returned value length is limited by the shared capture buffer size
+  controlled by "tune.ssl.capture-cipherlist-size" setting. Setting
+  <filter_option> allows to filter returned data. Accepted values:
+    0 : return the full list of extensions (default)
+    1 : exclude GREASE (RFC8701) values from the output
+
+  Example:
+      http-request set-header X-SSL-JA3 %[ssl_fc_protocol_hello_id],\
+          %[ssl_fc_cipherlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_extlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_eclist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_ecformats_bin,be2dec(-,1)]
+      acl is_malware req.fhdr(x-ssl-ja3),digest(md5),hex \
+          -f /path/to/file/with/malware-ja3.lst
+      http-request set-header X-Malware True if is_malware
+      http-request set-header X-Malware False if !is_malware
 
 ssl_fc_client_random : binary
   Returns the client random of the front connection when the incoming connection
@@ -19005,6 +19081,23 @@ ssl_fc_protocol : string
   Returns the name of the used protocol when the incoming connection was made
   over an SSL/TLS transport layer.
 
+ssl_fc_protocol_hello_id : integer
+  The version of the TLS protocol by which the client wishes to communicate
+  during the session as indicated in client hello message. This value can
+  return only if the value "tune.ssl.capture-cipherlist-size" is set greater
+  than 0.
+
+  Example:
+      http-request set-header X-SSL-JA3 %[ssl_fc_protocol_hello_id],\
+          %[ssl_fc_cipherlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_extlist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_eclist_bin(1),be2dec(-,2)],\
+          %[ssl_fc_ecformats_bin,be2dec(-,1)]
+      acl is_malware req.fhdr(x-ssl-ja3),digest(md5),hex \
+          -f /path/to/file/with/malware-ja3.lst
+      http-request set-header X-Malware True if is_malware
+      http-request set-header X-Malware False if !is_malware
+
 ssl_fc_unique_id : binary
   When the incoming connection was made over an SSL/TLS transport layer,
   returns the TLS unique ID as defined in RFC5929 section 3. The unique id
diff --git a/include/haproxy/ssl_utils.h b/include/haproxy/ssl_utils.h
index 9851e8a36..e14aaf1c6 100644
--- a/include/haproxy/ssl_utils.h
+++ b/include/haproxy/ssl_utils.h
@@ -40,6 +40,7 @@ int ssl_sock_get_dn_formatted(X509_NAME *a, const struct buffer *format, struct
 int ssl_sock_get_dn_oneline(X509_NAME *a, struct buffer *out);
 X509* ssl_sock_get_peer_certificate(SSL *ssl);
 unsigned int openssl_version_parser(const char *version);
+void exclude_tls_grease(char *input, int len, struct buffer *output);
 
 #endif /* _HAPROXY_SSL_UTILS_H */
 #endif /* USE_OPENSSL */
diff --git a/src/ssl_sample.c b/src/ssl_sample.c
index 9f041ad8e..aa9a547e4 100644
--- a/src/ssl_sample.c
+++ b/src/ssl_sample.c
@@ -1127,9 +1127,13 @@ smp_fetch_ssl_fc_sni(const struct arg *args, struct sample *smp, const char *kw,
 }
 #endif
 
+/* binary, returns tls client hello cipher list.
+ * Arguments: filter_option (0,1)
+ */
 static int
 smp_fetch_ssl_fc_cl_bin(const struct arg *args, struct sample *smp, const char *kw, void *private)
 {
+	struct buffer *smp_trash;
 	struct connection *conn;
 	struct ssl_capture *capture;
 	SSL *ssl;
@@ -1143,13 +1147,26 @@ smp_fetch_ssl_fc_cl_bin(const struct arg *args, struct sample *smp, const char *
 	if (!capture)
 		return 0;
 
-	smp->flags = SMP_F_VOL_TEST | SMP_F_CONST;
+	if (args[0].data.sint) {
+		smp_trash = get_trash_chunk();
+		exclude_tls_grease(capture->data + capture->ciphersuite_offset, capture->ciphersuite_len, smp_trash);
+		smp->data.u.str.area = smp_trash->area;
+		smp->data.u.str.data = smp_trash->data;
+		smp->flags = SMP_F_VOL_SESS;
+	}
+	else {
+		smp->data.u.str.area = capture->data + capture->ciphersuite_offset;
+		smp->data.u.str.data = capture->ciphersuite_len;
+		smp->flags = SMP_F_VOL_TEST | SMP_F_CONST;
+	}
+
 	smp->data.type = SMP_T_BIN;
-	smp->data.u.str.area = capture->data + capture->ciphersuite_offset;
-	smp->data.u.str.data = capture->ciphersuite_len;
 	return 1;
 }
 
+/* binary, returns tls client hello cipher list as hexadecimal string.
+ * Arguments: filter_option (0,1)
+ */
 static int
 smp_fetch_ssl_fc_cl_hex(const struct arg *args, struct sample *smp, const char *kw, void *private)
 {
@@ -1166,6 +1183,7 @@ smp_fetch_ssl_fc_cl_hex(const struct arg *args, struct sample *smp, const char *
 	return 1;
 }
 
+/* integer, returns xxh64 hash of tls client hello cipher list. */
 static int
 smp_fetch_ssl_fc_cl_xxh64(const struct arg *args, struct sample *smp, const char *kw, void *private)
 {
@@ -1214,6 +1232,28 @@ smp_fetch_ssl_fc_hsk_err(const struct arg *args, struct sample *smp, const char
 }
 
 static int
+smp_fetch_ssl_fc_protocol_hello_id(const struct arg *args, struct sample *smp, const char *kw, void *private)
+{
+	struct connection *conn;
+	struct ssl_capture *capture;
+	SSL *ssl;
+
+	conn = objt_conn(smp->sess->origin);
+	ssl = ssl_sock_get_ssl_object(conn);
+	if (!ssl)
+		return 0;
+
+	capture = SSL_get_ex_data(ssl, ssl_capture_ptr_index);
+	if (!capture)
+		return 0;
+
+	smp->flags = SMP_F_VOL_SESS;
+	smp->data.type = SMP_T_SINT;
+	smp->data.u.sint = capture->protocol_version;
+	return 1;
+}
+
+static int
 smp_fetch_ssl_fc_hsk_err_str(const struct arg *args, struct sample *smp, const char *kw, void *private)
 {
 	struct connection *conn;
@@ -1243,6 +1283,104 @@ smp_fetch_ssl_fc_hsk_err_str(const struct arg *args, struct sample *smp, const c
 	return 1;
 }
 
+/* binary, returns tls client hello extensions list.
+ * Arguments: filter_option (0,1)
+ */
+static int
+smp_fetch_ssl_fc_ext_bin(const struct arg *args, struct sample *smp, const char *kw, void *private)
+{
+	struct buffer *smp_trash;
+	struct connection *conn;
+	struct ssl_capture *capture;
+	SSL *ssl;
+
+	conn = objt_conn(smp->sess->origin);
+	ssl = ssl_sock_get_ssl_object(conn);
+	if (!ssl)
+		return 0;
+
+	capture = SSL_get_ex_data(ssl, ssl_capture_ptr_index);
+	if (!capture)
+		return 0;
+
+	if (args[0].data.sint) {
+		smp_trash = get_trash_chunk();
+		exclude_tls_grease(capture->data + capture->extensions_offset, capture->extensions_len, smp_trash);
+		smp->data.u.str.area = smp_trash->area;
+		smp->data.u.str.data = smp_trash->data;
+		smp->flags = SMP_F_VOL_SESS;
+	}
+	else {
+		smp->data.u.str.area = capture->data + capture->extensions_offset;
+		smp->data.u.str.data = capture->extensions_len;
+		smp->flags = SMP_F_VOL_TEST | SMP_F_CONST;
+	}
+
+	smp->data.type = SMP_T_BIN;
+	return 1;
+}
+
+/* binary, returns tls client hello supported elliptic curves.
+ * Arguments: filter_option (0,1)
+ */
+static int
+smp_fetch_ssl_fc_ecl_bin(const struct arg *args, struct sample *smp, const char *kw, void *private)
+{
+	struct buffer *smp_trash;
+	struct connection *conn;
+	struct ssl_capture *capture;
+	SSL *ssl;
+
+	conn = objt_conn(smp->sess->origin);
+	ssl = ssl_sock_get_ssl_object(conn);
+	if (!ssl)
+		return 0;
+
+	capture = SSL_get_ex_data(ssl, ssl_capture_ptr_index);
+	if (!capture)
+		return 0;
+
+	if (args[0].data.sint) {
+		smp_trash = get_trash_chunk();
+		exclude_tls_grease(capture->data + capture->ec_offset, capture->ec_len, smp_trash);
+		smp->data.u.str.area = smp_trash->area;
+		smp->data.u.str.data = smp_trash->data;
+		smp->flags = SMP_F_VOL_SESS;
+	}
+	else {
+		smp->data.u.str.area = capture->data + capture->ec_offset;
+		smp->data.u.str.data = capture->ec_len;
+		smp->flags = SMP_F_VOL_TEST | SMP_F_CONST;
+	}
+
+	smp->data.type = SMP_T_BIN;
+	return 1;
+}
+
+/* binary, returns tls client hello supported elliptic curve point formats */
+static int
+smp_fetch_ssl_fc_ecf_bin(const struct arg *args, struct sample *smp, const char *kw, void *private)
+{
+	struct connection *conn;
+	struct ssl_capture *capture;
+	SSL *ssl;
+
+	conn = objt_conn(smp->sess->origin);
+	ssl = ssl_sock_get_ssl_object(conn);
+	if (!ssl)
+		return 0;
+
+	capture = SSL_get_ex_data(ssl, ssl_capture_ptr_index);
+	if (!capture)
+		return 0;
+
+	smp->flags = SMP_F_VOL_TEST | SMP_F_CONST;
+	smp->data.type = SMP_T_BIN;
+	smp->data.u.str.area = capture->data + capture->ec_formats_offset;
+	smp->data.u.str.data = capture->ec_formats_len;
+	return 1;
+}
+
 /* Dump the SSL keylog, it only works with "tune.ssl.keylog 1" */
 #ifdef HAVE_SSL_KEYLOG
 static int smp_fetch_ssl_x_keylog(const struct arg *args, struct sample *smp, const char *kw, void *private)
@@ -1597,12 +1735,16 @@ static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
 #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
 	{ "ssl_fc_sni",             smp_fetch_ssl_fc_sni,         0,                   NULL,    SMP_T_STR,  SMP_USE_L5CLI },
 #endif
-	{ "ssl_fc_cipherlist_bin",  smp_fetch_ssl_fc_cl_bin,      0,                   NULL,    SMP_T_STR,  SMP_USE_L5CLI },
-	{ "ssl_fc_cipherlist_hex",  smp_fetch_ssl_fc_cl_hex,      0,                   NULL,    SMP_T_BIN,  SMP_USE_L5CLI },
-	{ "ssl_fc_cipherlist_str",  smp_fetch_ssl_fc_cl_str,      0,                   NULL,    SMP_T_STR,  SMP_USE_L5CLI },
+	{ "ssl_fc_cipherlist_bin",  smp_fetch_ssl_fc_cl_bin,      ARG1(0,SINT),        NULL,    SMP_T_STR,  SMP_USE_L5CLI },
+	{ "ssl_fc_cipherlist_hex",  smp_fetch_ssl_fc_cl_hex,      ARG1(0,SINT),        NULL,    SMP_T_BIN,  SMP_USE_L5CLI },
+	{ "ssl_fc_cipherlist_str",  smp_fetch_ssl_fc_cl_str,      ARG1(0,SINT),        NULL,    SMP_T_STR,  SMP_USE_L5CLI },
 	{ "ssl_fc_cipherlist_xxh",  smp_fetch_ssl_fc_cl_xxh64,    0,                   NULL,    SMP_T_SINT, SMP_USE_L5CLI },
 	{ "ssl_fc_hsk_err",         smp_fetch_ssl_fc_hsk_err,     0,                   NULL,    SMP_T_SINT, SMP_USE_L5CLI },
 	{ "ssl_fc_hsk_err_str",     smp_fetch_ssl_fc_hsk_err_str, 0,                   NULL,    SMP_T_STR, SMP_USE_L5CLI },
+	{ "ssl_fc_protocol_hello_id",smp_fetch_ssl_fc_protocol_hello_id,0,             NULL,    SMP_T_SINT, SMP_USE_L5CLI },
+	{ "ssl_fc_extlist_bin",     smp_fetch_ssl_fc_ext_bin,     ARG1(0,SINT),        NULL,    SMP_T_STR,  SMP_USE_L5CLI },
+	{ "ssl_fc_eclist_bin",      smp_fetch_ssl_fc_ecl_bin,     ARG1(0,SINT),        NULL,    SMP_T_STR,  SMP_USE_L5CLI },
+	{ "ssl_fc_ecformats_bin",   smp_fetch_ssl_fc_ecf_bin,     0,                   NULL,    SMP_T_STR,  SMP_USE_L5CLI },
 
 /* SSL server certificate fetches */
 	{ "ssl_s_der",              smp_fetch_ssl_x_der,          0,                   NULL,    SMP_T_BIN,  SMP_USE_L5CLI },
diff --git a/src/ssl_utils.c b/src/ssl_utils.c
index 578212182..35c06f73d 100644
--- a/src/ssl_utils.c
+++ b/src/ssl_utils.c
@@ -397,3 +397,22 @@ error:
 	return 0;
 
 }
+
+/* Exclude GREASE (RFC8701) values from input buffer */
+void exclude_tls_grease(char *input, int len, struct buffer *output)
+{
+	int ptr = 0;
+
+	while (ptr < len - 1) {
+		if (input[ptr] != input[ptr+1] || (input[ptr] & 0x0f) != 0x0a) {
+			if (output->data <= output->size - 2) {
+				memcpy(output->area + output->data, input + ptr, 2);
+				output->data += 2;
+			} else
+				break;
+		}
+		ptr += 2;
+	}
+	if (output->size - output->data > 0 && len - ptr > 0)
+		output->area[output->data++] = input[ptr];
+}