Enable to get fragment_len includes metadata_adder

This patch allows to get correct fragment size includes metadata_adder.

Current implementaion automatically allocates extra bytes for the metadata_adder
in alloc_buffer, and then, no information about the extra bytes will be returned
to the api caller side. It's too confusable because callers couldn't know how size they
assumes as the fragment size.

To be easy to find out the size infomation, this patch adds "frag_adder_size"
variable into fragment metadata and also some functions to get fragment size.

The definitions of these size infomation are here,

fragment_meta:
- size-> raw data size used to encode/fragment_to_string
- frag_adder_size-> metadata_adder of backend specification

And the definitions of functions are here,

- get_fragment_size:
  -> return sizeof(fragument_header) + size + frag_adder_size

- get_fragment_buffer_size:
  -> return size + frag_adder_size

- get_fragment_payload_size:
  -> return size

By using these function above, users could get the size information
directly from fragments. It results in enabling to return fragment_len
to the caller side easily.

8 files changed, 71 insertions, 34 deletions

diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h
index 4bd2a59..f811555 100644
--- a/include/erasurecode/erasurecode.h
+++ b/include/erasurecode/erasurecode.h
@@ -247,6 +247,7 @@ fragment_metadata
 {
     uint32_t    idx;                /* 4 */
     uint32_t    size;               /* 4 */
+    uint32_t    frag_adder_size;    /* 4 */
     uint64_t    orig_data_size;     /* 8 */
     uint8_t     chksum_type;        /* 1 */
     uint32_t    chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */
diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h
index 23b3ac8..f7eefd7 100644
--- a/include/erasurecode/erasurecode_helpers.h
+++ b/include/erasurecode/erasurecode_helpers.h
@@ -45,12 +45,12 @@
 
 typedef struct __attribute__((__packed__)) fragment_header_s
 {
-    fragment_metadata_t meta;   /* 55 bytes */
+    fragment_metadata_t meta;   /* 59 bytes */
     uint32_t            magic;  /*  4 bytes */
     uint32_t            libec_version; /* 4 bytes */
     // We must be aligned to 16-byte boundaries
     // So, size this array accordingly
-    uint8_t             aligned_padding[1];
+    uint8_t             aligned_padding[13];
 } fragment_header_t;
 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
@@ -119,7 +119,7 @@ void init_fragment_header(char *buf)
 void *alloc_zeroed_buffer(int size);
 void *alloc_and_set_buffer(int size, int value);
 void *check_and_free_buffer(void *buf);
-char *alloc_fragment_buffer(ec_backend_t instance, int size);
+char *alloc_fragment_buffer(int size);
 int free_fragment_buffer(char *buf);
 void *get_aligned_buffer16(int size);
 int get_aligned_data_size(ec_backend_t instance, int data_len);
@@ -135,6 +135,9 @@ int set_fragment_idx(char *buf, int idx);
 int get_fragment_idx(char *buf);
 int set_fragment_payload_size(char *buf, int size);
 int get_fragment_payload_size(char *buf);
+int set_fragment_adder_size(char *buf, int size);
+int get_fragment_adder_size(char *buf);
+int get_fragment_buffer_size(char *buf);
 int set_orig_data_size(char *buf, int orig_data_size);
 int get_orig_data_size(char *buf);
 int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize);
diff --git a/include/erasurecode/erasurecode_preprocessing.h b/include/erasurecode/erasurecode_preprocessing.h
index 0b479c6..7ca891d 100644
--- a/include/erasurecode/erasurecode_preprocessing.h
+++ b/include/erasurecode/erasurecode_preprocessing.h
@@ -37,7 +37,6 @@ int prepare_fragments_for_encode(
         int *blocksize);
 
 int prepare_fragments_for_decode(
-        ec_backend_t instance,
         int k, int m,
         char **data, char **parity,
         int *missing_idxs,
diff --git a/src/erasurecode.c b/src/erasurecode.c
index 5bba185..0f626b5 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -631,7 +631,7 @@ int liberasurecode_decode(int desc,
      * (realloc_bm).
      *
      */
-    ret = prepare_fragments_for_decode(instance, k, m,
+    ret = prepare_fragments_for_decode(k, m,
                                        data, parity, missing_idxs, 
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
@@ -799,7 +799,7 @@ int liberasurecode_reconstruct_fragment(int desc,
      * It passes back a bitmap telling us which buffers need to be freed by
      * us (realloc_bm).
      */
-    ret = prepare_fragments_for_decode(instance, k, m, data, parity, missing_idxs,
+    ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
     if (ret < 0) {
diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c
index f0f3fe0..05bc838 100644
--- a/src/erasurecode_helpers.c
+++ b/src/erasurecode_helpers.c
@@ -116,16 +116,11 @@ void * check_and_free_buffer(void * buf)
     return NULL;
 }
 
-char *alloc_fragment_buffer(ec_backend_t instance, int size)
+char *alloc_fragment_buffer(int size)
 {
     char *buf;
     fragment_header_t *header = NULL;
 
-    if (NULL != instance){
-         /* Account for any custom metadata the backend wants to add in data_len */
-         size += instance->common.metadata_adder;
-    }
-
     size += sizeof(fragment_header_t);
     buf = get_aligned_buffer16(size);
 
@@ -168,13 +163,11 @@ int free_fragment_buffer(char *buf)
  */
 uint64_t get_fragment_size(char *buf)
 {
-    fragment_header_t *header = NULL;
 
     if (NULL == buf)
         return -1;
 
-    header = (fragment_header_t *) buf;
-    return (header->meta.size + sizeof(fragment_header_t));
+    return get_fragment_buffer_size(buf) + sizeof(fragment_header_t);
  }
 
 /**
@@ -331,6 +324,47 @@ int get_fragment_payload_size(char *buf)
     return header->meta.size;
 }
 
+int set_fragment_adder_size(char *buf, int size)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (set adder size)!");
+        return -1;
+    }
+
+    header->meta.frag_adder_size = size;
+
+    return 0;
+}
+
+int get_fragment_adder_size(char *buf)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (get adder size)!");
+        return -1;
+    }
+
+    return header->meta.frag_adder_size;
+}
+
+int get_fragment_buffer_size(char *buf)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (get size)!");
+        return -1;
+    }
+
+    return header->meta.size + header->meta.frag_adder_size;
+}
+
 int set_orig_data_size(char *buf, int orig_data_size)
 {
     fragment_header_t *header = (fragment_header_t *) buf;
diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c
index 9ec68c7..435325e 100644
--- a/src/erasurecode_postprocessing.c
+++ b/src/erasurecode_postprocessing.c
@@ -41,7 +41,8 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
     set_fragment_payload_size(fragment, blocksize);
     set_backend_id(fragment, be->common.id);
     set_backend_version(fragment, be->common.ec_backend_version);
-    
+    set_fragment_adder_size(fragment, be->common.metadata_adder);
+
     if (add_chksum) {
         set_checksum(ct, fragment, blocksize);
     }
diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c
index 7723569..9666b9d 100644
--- a/src/erasurecode_preprocessing.c
+++ b/src/erasurecode_preprocessing.c
@@ -40,16 +40,17 @@ int prepare_fragments_for_encode(ec_backend_t instance,
     int i, ret = 0;
     int data_len;           /* data len to write to fragment headers */
     int aligned_data_len;   /* EC algorithm compatible data length */
-    int bsize = 0;
+    int buffer_size, payload_size = 0;
 
     /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
     data_len = orig_data_size;
     aligned_data_len = get_aligned_data_size(instance, orig_data_size);
-    *blocksize = bsize = (aligned_data_len / k);
+    *blocksize = payload_size = (aligned_data_len / k);
+    buffer_size = payload_size + instance->common.metadata_adder;
 
     for (i = 0; i < k; i++) {
-        int payload_size = data_len > bsize ? bsize : data_len;
-        char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
+        int copy_size = data_len > payload_size ? payload_size : data_len;
+        char *fragment = (char *) alloc_fragment_buffer(buffer_size);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -59,15 +60,15 @@ int prepare_fragments_for_encode(ec_backend_t instance,
         encoded_data[i] = get_data_ptr_from_fragment(fragment);
       
         if (data_len > 0) {
-            memcpy(encoded_data[i], orig_data, payload_size);
+            memcpy(encoded_data[i], orig_data, copy_size);
         }
 
-        orig_data += payload_size;
-        data_len -= payload_size;
+        orig_data += copy_size;
+        data_len -= copy_size;
     }
 
     for (i = 0; i < m; i++) {
-        char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
+        char *fragment = (char *) alloc_fragment_buffer(buffer_size);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -108,7 +109,6 @@ out_error:
  * so in the failure case.
  */
 int prepare_fragments_for_decode(
-        ec_backend_t instance,
         int k, int m,
         char **data, char **parity,
         int  *missing_idxs,
@@ -136,16 +136,14 @@ int prepare_fragments_for_decode(
          * 'data_list'
          */
         if (NULL == data[i]) {
-            data[i] = alloc_fragment_buffer(
-                instance, fragment_size - sizeof(fragment_header_t));
+            data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
             if (NULL == data[i]) {
                 log_error("Could not allocate data buffer!");
                 return -1;
             }
             *realloc_bm = *realloc_bm | (1 << i);
         } else if (!is_addr_aligned((unsigned long)data[i], 16)) {
-            char *tmp_buf = alloc_fragment_buffer(
-                instance, fragment_size - sizeof(fragment_header_t));
+            char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
             if (NULL == tmp_buf) {
                 log_error("Could not allocate temp buffer!");
                 return -1;
@@ -177,16 +175,14 @@ int prepare_fragments_for_decode(
          * DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
          */
         if (NULL == parity[i]) {
-            parity[i] = alloc_fragment_buffer(
-                instance, fragment_size-sizeof(fragment_header_t));
+            parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
             if (NULL == parity[i]) {
                 log_error("Could not allocate parity buffer!");
                 return -1;
             }
             *realloc_bm = *realloc_bm | (1 << (k + i));
         } else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
-            char *tmp_buf = alloc_fragment_buffer(
-                instance, fragment_size-sizeof(fragment_header_t));
+            char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
             if (NULL == tmp_buf) {
                 log_error("Could not allocate temp buffer!");
                 return -1;
diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c
index 04d06cf..3d9ef6c 100644
--- a/test/liberasurecode_test.c
+++ b/test/liberasurecode_test.c
@@ -592,8 +592,11 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
     int num_avail_frags = 0;
     char *orig_data_ptr = NULL;
     int remaining = 0;
+    ec_backend_t be = NULL;
 
     desc = liberasurecode_instance_create(be_id, args);
+    be = liberasurecode_backend_instance_get_by_desc(desc);
+
     if (-EBACKENDNOTAVAIL == desc) {
         fprintf (stderr, "Backend library not available!\n");
         return;
@@ -614,7 +617,7 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
         assert(header != NULL);
         fragment_metadata_t metadata = header->meta;
         assert(metadata.idx == i);
-        assert(metadata.size == encoded_fragment_len - frag_header_size);
+        assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.metadata_adder);
         assert(metadata.orig_data_size == orig_data_size);
         char *data_ptr = frag + frag_header_size;
         int cmp_size = remaining >= metadata.size ? metadata.size : remaining;