ldb: Add ldb_unpack_data_only_attr_list_flags()

This function allows us to control allocation of memory during parse
of the packed ldb data.

This in turn can have an important performance impact as each
small allocation can have a large overhead

Signed-off-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>

4 files changed, 138 insertions, 24 deletions

diff --git a/lib/ldb/ABI/ldb-1.1.26.sigs b/lib/ldb/ABI/ldb-1.1.26.sigs
index 3f33df96f74..4fa30d88ec9 100644
--- a/lib/ldb/ABI/ldb-1.1.26.sigs
+++ b/lib/ldb/ABI/ldb-1.1.26.sigs
@@ -254,6 +254,7 @@ ldb_transaction_prepare_commit: int (struct ldb_context *)
 ldb_transaction_start: int (struct ldb_context *)
 ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *)
 ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *)
+ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *)
 ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *)
 ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *)
 ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *)
diff --git a/lib/ldb/common/ldb_pack.c b/lib/ldb/common/ldb_pack.c
index 7970b9d3ecb..cc3a5527583 100644
--- a/lib/ldb/common/ldb_pack.c
+++ b/lib/ldb/common/ldb_pack.c
@@ -211,20 +211,20 @@ static bool ldb_consume_element_data(uint8_t **pp, size_t *premaining)
 	return true;
 }
 
+
 /*
  * Unpack a ldb message from a linear buffer in ldb_val
  *
  * Providing a list of attributes to this function allows selective unpacking.
  * Giving a NULL list (or a list_size of 0) unpacks all the attributes.
- *
- * Free with ldb_unpack_data_free()
  */
-int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
-				   const struct ldb_val *data,
-				   struct ldb_message *message,
-				   const char * const *list,
-				   unsigned int list_size,
-				   unsigned int *nb_elements_in_db)
+int ldb_unpack_data_only_attr_list_flags(struct ldb_context *ldb,
+					 const struct ldb_val *data,
+					 struct ldb_message *message,
+					 const char * const *list,
+					 unsigned int list_size,
+					 unsigned int flags,
+					 unsigned int *nb_elements_in_db)
 {
 	uint8_t *p;
 	size_t remaining;
@@ -271,10 +271,14 @@ int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
 			errno = EIO;
 			goto failed;
 		}
-		message->dn = ldb_dn_new(message, ldb, (char *)p);
-		if (message->dn == NULL) {
-			errno = ENOMEM;
-			goto failed;
+		if (flags & LDB_UNPACK_DATA_FLAG_NO_DN) {
+			message->dn = NULL;
+		} else {
+			message->dn = ldb_dn_new(message, ldb, (char *)p);
+			if (message->dn == NULL) {
+				errno = ENOMEM;
+				goto failed;
+			}
 		}
 		/*
 		 * Redundant: by definition, remaining must be more
@@ -373,11 +377,15 @@ int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
 			}
 		}
 		element = &message->elements[nelem];
-		element->name = talloc_memdup(message->elements, attr, attr_len+1);
+		if (flags & LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC) {
+			element->name = attr;
+		} else {
+			element->name = talloc_memdup(message->elements, attr, attr_len+1);
 
-		if (element->name == NULL) {
-			errno = ENOMEM;
-			goto failed;
+			if (element->name == NULL) {
+				errno = ENOMEM;
+				goto failed;
+			}
 		}
 		element->flags = 0;
 
@@ -422,15 +430,18 @@ int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
 			}
 
 			element->values[j].length = len;
-			element->values[j].data = talloc_size(element->values, len+1);
-			if (element->values[j].data == NULL) {
-				errno = ENOMEM;
-				goto failed;
+			if (flags & LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC) {
+				element->values[j].data = p + 4;
+			} else {
+				element->values[j].data = talloc_size(element->values, len+1);
+				if (element->values[j].data == NULL) {
+					errno = ENOMEM;
+					goto failed;
+				}
+				memcpy(element->values[j].data, p + 4,
+				       len);
+				element->values[j].data[len] = 0;
 			}
-			memcpy(element->values[j].data, p + 4,
-			       len);
-			element->values[j].data[len] = 0;
-
 			remaining -= len;
 			p += len+4+1;
 		}
@@ -463,6 +474,30 @@ failed:
 	return -1;
 }
 
+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ *
+ * Providing a list of attributes to this function allows selective unpacking.
+ * Giving a NULL list (or a list_size of 0) unpacks all the attributes.
+ *
+ * Free with ldb_unpack_data_free()
+ */
+int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
+				   const struct ldb_val *data,
+				   struct ldb_message *message,
+				   const char * const *list,
+				   unsigned int list_size,
+				   unsigned int *nb_elements_in_db)
+{
+	return ldb_unpack_data_only_attr_list_flags(ldb,
+						    data,
+						    message,
+						    list,
+						    list_size,
+						    0,
+						    nb_elements_in_db);
+}
+
 int ldb_unpack_data(struct ldb_context *ldb,
 		    const struct ldb_val *data,
 		    struct ldb_message *message)
diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h
index c6a24d35e53..1c48590a381 100644
--- a/lib/ldb/include/ldb_module.h
+++ b/lib/ldb/include/ldb_module.h
@@ -390,6 +390,12 @@ int ldb_register_extended_match_rule(struct ldb_context *ldb,
 int ldb_pack_data(struct ldb_context *ldb,
 		  const struct ldb_message *message,
 		  struct ldb_val *data);
+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ *
+ * Providing a list of attributes to this function allows selective unpacking.
+ * Giving a NULL list (or a list_size of 0) unpacks all the attributes.
+ */
 int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
 				   const struct ldb_val *data,
 				   struct ldb_message *message,
@@ -399,5 +405,27 @@ int ldb_unpack_data_only_attr_list(struct ldb_context *ldb,
 int ldb_unpack_data(struct ldb_context *ldb,
 		    const struct ldb_val *data,
 		    struct ldb_message *message);
+/*
+ * Unpack a ldb message from a linear buffer in ldb_val
+ *
+ * Providing a list of attributes to this function allows selective unpacking.
+ * Giving a NULL list (or a list_size of 0) unpacks all the attributes.
+ *
+ * Flags allow control of allocation, so that if
+ * LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC is specified, then values are
+ * not allocate, instead they point into the supplier constant buffer.
+ *
+ * Likewise if LDB_UNPACK_DATA_FLAG_NO_DN is specified, the DN is omitted.
+ */
+int ldb_unpack_data_only_attr_list_flags(struct ldb_context *ldb,
+					 const struct ldb_val *data,
+					 struct ldb_message *message,
+					 const char * const *list,
+					 unsigned int list_size,
+					 unsigned int flags,
+					 unsigned int *nb_elements_in_db);
+
+#define LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC 0x0001
+#define LDB_UNPACK_DATA_FLAG_NO_DN         0x0002
 
 #endif
diff --git a/source4/torture/ldb/ldb.c b/source4/torture/ldb/ldb.c
index 7ea9726d520..f7f04db0242 100644
--- a/source4/torture/ldb/ldb.c
+++ b/source4/torture/ldb/ldb.c
@@ -1102,6 +1102,54 @@ static bool torture_ldb_unpack(struct torture_context *torture)
 	return true;
 }
 
+static bool torture_ldb_unpack_flags(struct torture_context *torture)
+{
+	TALLOC_CTX *mem_ctx = talloc_new(torture);
+	struct ldb_context *ldb;
+	struct ldb_val data = data_blob_const(dda1d01d_bin, sizeof(dda1d01d_bin));
+	struct ldb_message *msg = ldb_msg_new(mem_ctx);
+	const char *ldif_text = dda1d01d_ldif;
+	struct ldb_ldif ldif;
+	unsigned int nb_elements_in_db;
+
+	ldb = samba_ldb_init(mem_ctx, torture->ev, NULL, NULL, NULL);
+	torture_assert(torture,
+		       ldb != NULL,
+		       "Failed to init ldb");
+
+	torture_assert_int_equal(torture,
+				 ldb_unpack_data_only_attr_list_flags(ldb, &data,
+								      msg,
+								      NULL, 0,
+								      LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC,
+								      &nb_elements_in_db),
+				 0,
+				 "ldb_unpack_data failed");
+
+	ldif.changetype = LDB_CHANGETYPE_NONE;
+	ldif.msg = msg;
+	ldif_text = ldb_ldif_write_string(ldb, mem_ctx, &ldif);
+
+	torture_assert_int_equal(torture,
+				 strcmp(ldif_text, dda1d01d_ldif), 0,
+				 "ldif form differs from binary form");
+
+	torture_assert_int_equal(torture,
+				 ldb_unpack_data_only_attr_list_flags(ldb, &data,
+								      msg,
+								      NULL, 0,
+								      LDB_UNPACK_DATA_FLAG_NO_DN,
+								      &nb_elements_in_db),
+				 0,
+				 "ldb_unpack_data failed");
+
+	torture_assert(torture,
+		       msg->dn == NULL,
+		       "msg->dn should be NULL");
+
+	return true;
+}
+
 static bool torture_ldb_parse_ldif(struct torture_context *torture)
 {
 	TALLOC_CTX *mem_ctx = talloc_new(torture);
@@ -1265,6 +1313,8 @@ struct torture_suite *torture_ldb(TALLOC_CTX *mem_ctx)
 	torture_suite_add_simple_test(suite, "dn", torture_ldb_dn);
 	torture_suite_add_simple_test(suite, "unpack-data",
 				      torture_ldb_unpack);
+	torture_suite_add_simple_test(suite, "unpack-data-flags",
+				      torture_ldb_unpack_flags);
 	torture_suite_add_simple_test(suite, "parse-ldif",
 				      torture_ldb_parse_ldif);
 	torture_suite_add_simple_test(suite, "unpack-data-only-attr-list",