Move backend metadata adding to fragment allocation

On the first consideration[1], metadata_adder is defined as a extra byte
size for "each" fragment. However, current implementation is an element
affects to data_len. (i.e. aligned_data_size for original segment data)

We should make metadata_adder to be a fixed value against to each fragment,
otherwise the extra bytes for the fragments will have flexible length depends
on "K". It will be quite complex for backend implementation to know "How large
bytes the raw data size is", "How large bytes the backend could use as extra
bytes for each fragment".

1: https://bitbucket.org/tsg-/liberasurecode/commits/032b57d9b1c7aadc547fccbacf88af786c9067e7?at=master

5 files changed, 22 insertions, 14 deletions

diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h
index 9c4e50d..23b3ac8 100644
--- a/include/erasurecode/erasurecode_helpers.h
+++ b/include/erasurecode/erasurecode_helpers.h
@@ -119,7 +119,7 @@ void init_fragment_header(char *buf)
 void *alloc_zeroed_buffer(int size);
 void *alloc_and_set_buffer(int size, int value);
 void *check_and_free_buffer(void *buf);
-char *alloc_fragment_buffer(int size);
+char *alloc_fragment_buffer(ec_backend_t instance, int size);
 int free_fragment_buffer(char *buf);
 void *get_aligned_buffer16(int size);
 int get_aligned_data_size(ec_backend_t instance, int data_len);
diff --git a/include/erasurecode/erasurecode_preprocessing.h b/include/erasurecode/erasurecode_preprocessing.h
index 7ca891d..0b479c6 100644
--- a/include/erasurecode/erasurecode_preprocessing.h
+++ b/include/erasurecode/erasurecode_preprocessing.h
@@ -37,6 +37,7 @@ int prepare_fragments_for_encode(
         int *blocksize);
 
 int prepare_fragments_for_decode(
+        ec_backend_t instance,
         int k, int m,
         char **data, char **parity,
         int *missing_idxs,
diff --git a/src/erasurecode.c b/src/erasurecode.c
index 775f37c..5bba185 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -631,8 +631,8 @@ int liberasurecode_decode(int desc,
      * (realloc_bm).
      *
      */
-    ret = prepare_fragments_for_decode(k, m,
-                                       data, parity, missing_idxs,
+    ret = prepare_fragments_for_decode(instance, k, m,
+                                       data, parity, missing_idxs, 
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
     if (ret < 0) {
@@ -799,7 +799,7 @@ int liberasurecode_reconstruct_fragment(int desc,
      * It passes back a bitmap telling us which buffers need to be freed by
      * us (realloc_bm).
      */
-    ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
+    ret = prepare_fragments_for_decode(instance, k, m, data, parity, missing_idxs,
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
     if (ret < 0) {
diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c
index 54e711b..f0f3fe0 100644
--- a/src/erasurecode_helpers.c
+++ b/src/erasurecode_helpers.c
@@ -116,11 +116,16 @@ void * check_and_free_buffer(void * buf)
     return NULL;
 }
 
-char *alloc_fragment_buffer(int size)
+char *alloc_fragment_buffer(ec_backend_t instance, int size)
 {
     char *buf;
     fragment_header_t *header = NULL;
 
+    if (NULL != instance){
+         /* Account for any custom metadata the backend wants to add in data_len */
+         size += instance->common.metadata_adder;
+    }
+
     size += sizeof(fragment_header_t);
     buf = get_aligned_buffer16(size);
 
@@ -188,9 +193,6 @@ int get_aligned_data_size(ec_backend_t instance, int data_len)
     int alignment_multiple;
     int aligned_size = 0;
 
-    /* Account for any custom metadata the backend wants to add in data_len */
-    data_len += instance->common.metadata_adder;
-
     /*
      * For Cauchy reed-solomon align to k*word_size*packet_size
      * For Vandermonde reed-solomon and flat-XOR, align to k*word_size
diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c
index b55f635..7723569 100644
--- a/src/erasurecode_preprocessing.c
+++ b/src/erasurecode_preprocessing.c
@@ -49,7 +49,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
 
     for (i = 0; i < k; i++) {
         int payload_size = data_len > bsize ? bsize : data_len;
-        char *fragment = (char *) alloc_fragment_buffer(bsize);
+        char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -67,7 +67,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
     }
 
     for (i = 0; i < m; i++) {
-        char *fragment = (char *) alloc_fragment_buffer(bsize);
+        char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -108,6 +108,7 @@ out_error:
  * so in the failure case.
  */
 int prepare_fragments_for_decode(
+        ec_backend_t instance,
         int k, int m,
         char **data, char **parity,
         int  *missing_idxs,
@@ -135,14 +136,16 @@ int prepare_fragments_for_decode(
          * 'data_list'
          */
         if (NULL == data[i]) {
-            data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
+            data[i] = alloc_fragment_buffer(
+                instance, fragment_size - sizeof(fragment_header_t));
             if (NULL == data[i]) {
                 log_error("Could not allocate data buffer!");
                 return -1;
             }
             *realloc_bm = *realloc_bm | (1 << i);
         } else if (!is_addr_aligned((unsigned long)data[i], 16)) {
-            char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
+            char *tmp_buf = alloc_fragment_buffer(
+                instance, fragment_size - sizeof(fragment_header_t));
             if (NULL == tmp_buf) {
                 log_error("Could not allocate temp buffer!");
                 return -1;
@@ -174,14 +177,16 @@ int prepare_fragments_for_decode(
          * DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
          */
         if (NULL == parity[i]) {
-            parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
+            parity[i] = alloc_fragment_buffer(
+                instance, fragment_size-sizeof(fragment_header_t));
             if (NULL == parity[i]) {
                 log_error("Could not allocate parity buffer!");
                 return -1;
             }
             *realloc_bm = *realloc_bm | (1 << (k + i));
         } else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
-            char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
+            char *tmp_buf = alloc_fragment_buffer(
+                instance, fragment_size-sizeof(fragment_header_t));
             if (NULL == tmp_buf) {
                 log_error("Could not allocate temp buffer!");
                 return -1;