Move backend metadata adding to fragment allocation

On the first consideration[1], metadata_adder is defined as a extra byte
size for "each" fragment. However, current implementation is an element
affects to data_len. (i.e. aligned_data_size for original segment data)

We should make metadata_adder to be a fixed value against to each fragment,
otherwise the extra bytes for the fragments will have flexible length depends
on "K". It will be quite complex for backend implementation to know "How large
bytes the raw data size is", "How large bytes the backend could use as extra
bytes for each fragment".

1: https://bitbucket.org/tsg-/liberasurecode/commits/032b57d9b1c7aadc547fccbacf88af786c9067e7?at=master

3 files changed, 20 insertions, 13 deletions

diff --git a/src/erasurecode.c b/src/erasurecode.c
index 775f37c..5bba185 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -631,8 +631,8 @@ int liberasurecode_decode(int desc,
      * (realloc_bm).
      *
      */
-    ret = prepare_fragments_for_decode(k, m,
-                                       data, parity, missing_idxs,
+    ret = prepare_fragments_for_decode(instance, k, m,
+                                       data, parity, missing_idxs, 
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
     if (ret < 0) {
@@ -799,7 +799,7 @@ int liberasurecode_reconstruct_fragment(int desc,
      * It passes back a bitmap telling us which buffers need to be freed by
      * us (realloc_bm).
      */
-    ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
+    ret = prepare_fragments_for_decode(instance, k, m, data, parity, missing_idxs,
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
     if (ret < 0) {
diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c
index 54e711b..f0f3fe0 100644
--- a/src/erasurecode_helpers.c
+++ b/src/erasurecode_helpers.c
@@ -116,11 +116,16 @@ void * check_and_free_buffer(void * buf)
     return NULL;
 }
 
-char *alloc_fragment_buffer(int size)
+char *alloc_fragment_buffer(ec_backend_t instance, int size)
 {
     char *buf;
     fragment_header_t *header = NULL;
 
+    if (NULL != instance){
+         /* Account for any custom metadata the backend wants to add in data_len */
+         size += instance->common.metadata_adder;
+    }
+
     size += sizeof(fragment_header_t);
     buf = get_aligned_buffer16(size);
 
@@ -188,9 +193,6 @@ int get_aligned_data_size(ec_backend_t instance, int data_len)
     int alignment_multiple;
     int aligned_size = 0;
 
-    /* Account for any custom metadata the backend wants to add in data_len */
-    data_len += instance->common.metadata_adder;
-
     /*
      * For Cauchy reed-solomon align to k*word_size*packet_size
      * For Vandermonde reed-solomon and flat-XOR, align to k*word_size
diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c
index b55f635..7723569 100644
--- a/src/erasurecode_preprocessing.c
+++ b/src/erasurecode_preprocessing.c
@@ -49,7 +49,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
 
     for (i = 0; i < k; i++) {
         int payload_size = data_len > bsize ? bsize : data_len;
-        char *fragment = (char *) alloc_fragment_buffer(bsize);
+        char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -67,7 +67,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
     }
 
     for (i = 0; i < m; i++) {
-        char *fragment = (char *) alloc_fragment_buffer(bsize);
+        char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -108,6 +108,7 @@ out_error:
  * so in the failure case.
  */
 int prepare_fragments_for_decode(
+        ec_backend_t instance,
         int k, int m,
         char **data, char **parity,
         int  *missing_idxs,
@@ -135,14 +136,16 @@ int prepare_fragments_for_decode(
          * 'data_list'
          */
         if (NULL == data[i]) {
-            data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
+            data[i] = alloc_fragment_buffer(
+                instance, fragment_size - sizeof(fragment_header_t));
             if (NULL == data[i]) {
                 log_error("Could not allocate data buffer!");
                 return -1;
             }
             *realloc_bm = *realloc_bm | (1 << i);
         } else if (!is_addr_aligned((unsigned long)data[i], 16)) {
-            char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
+            char *tmp_buf = alloc_fragment_buffer(
+                instance, fragment_size - sizeof(fragment_header_t));
             if (NULL == tmp_buf) {
                 log_error("Could not allocate temp buffer!");
                 return -1;
@@ -174,14 +177,16 @@ int prepare_fragments_for_decode(
          * DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
          */
         if (NULL == parity[i]) {
-            parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
+            parity[i] = alloc_fragment_buffer(
+                instance, fragment_size-sizeof(fragment_header_t));
             if (NULL == parity[i]) {
                 log_error("Could not allocate parity buffer!");
                 return -1;
             }
             *realloc_bm = *realloc_bm | (1 << (k + i));
         } else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
-            char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
+            char *tmp_buf = alloc_fragment_buffer(
+                instance, fragment_size-sizeof(fragment_header_t));
             if (NULL == tmp_buf) {
                 log_error("Could not allocate temp buffer!");
                 return -1;