Enable to get fragment_len includes metadata_adder

This patch allows to get correct fragment size includes metadata_adder.

Current implementaion automatically allocates extra bytes for the metadata_adder
in alloc_buffer, and then, no information about the extra bytes will be returned
to the api caller side. It's too confusable because callers couldn't know how size they
assumes as the fragment size.

To be easy to find out the size infomation, this patch adds "frag_adder_size"
variable into fragment metadata and also some functions to get fragment size.

The definitions of these size infomation are here,

fragment_meta:
- size-> raw data size used to encode/fragment_to_string
- frag_adder_size-> metadata_adder of backend specification

And the definitions of functions are here,

- get_fragment_size:
  -> return sizeof(fragument_header) + size + frag_adder_size

- get_fragment_buffer_size:
  -> return size + frag_adder_size

- get_fragment_payload_size:
  -> return size

By using these function above, users could get the size information
directly from fragments. It results in enabling to return fragment_len
to the caller side easily.

4 files changed, 60 insertions, 29 deletions

diff --git a/src/erasurecode.c b/src/erasurecode.c
index 5bba185..0f626b5 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -631,7 +631,7 @@ int liberasurecode_decode(int desc,
      * (realloc_bm).
      *
      */
-    ret = prepare_fragments_for_decode(instance, k, m,
+    ret = prepare_fragments_for_decode(k, m,
                                        data, parity, missing_idxs, 
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
@@ -799,7 +799,7 @@ int liberasurecode_reconstruct_fragment(int desc,
      * It passes back a bitmap telling us which buffers need to be freed by
      * us (realloc_bm).
      */
-    ret = prepare_fragments_for_decode(instance, k, m, data, parity, missing_idxs,
+    ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
     if (ret < 0) {
diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c
index f0f3fe0..05bc838 100644
--- a/src/erasurecode_helpers.c
+++ b/src/erasurecode_helpers.c
@@ -116,16 +116,11 @@ void * check_and_free_buffer(void * buf)
     return NULL;
 }
 
-char *alloc_fragment_buffer(ec_backend_t instance, int size)
+char *alloc_fragment_buffer(int size)
 {
     char *buf;
     fragment_header_t *header = NULL;
 
-    if (NULL != instance){
-         /* Account for any custom metadata the backend wants to add in data_len */
-         size += instance->common.metadata_adder;
-    }
-
     size += sizeof(fragment_header_t);
     buf = get_aligned_buffer16(size);
 
@@ -168,13 +163,11 @@ int free_fragment_buffer(char *buf)
  */
 uint64_t get_fragment_size(char *buf)
 {
-    fragment_header_t *header = NULL;
 
     if (NULL == buf)
         return -1;
 
-    header = (fragment_header_t *) buf;
-    return (header->meta.size + sizeof(fragment_header_t));
+    return get_fragment_buffer_size(buf) + sizeof(fragment_header_t);
  }
 
 /**
@@ -331,6 +324,47 @@ int get_fragment_payload_size(char *buf)
     return header->meta.size;
 }
 
+int set_fragment_adder_size(char *buf, int size)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (set adder size)!");
+        return -1;
+    }
+
+    header->meta.frag_adder_size = size;
+
+    return 0;
+}
+
+int get_fragment_adder_size(char *buf)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (get adder size)!");
+        return -1;
+    }
+
+    return header->meta.frag_adder_size;
+}
+
+int get_fragment_buffer_size(char *buf)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (get size)!");
+        return -1;
+    }
+
+    return header->meta.size + header->meta.frag_adder_size;
+}
+
 int set_orig_data_size(char *buf, int orig_data_size)
 {
     fragment_header_t *header = (fragment_header_t *) buf;
diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c
index 9ec68c7..435325e 100644
--- a/src/erasurecode_postprocessing.c
+++ b/src/erasurecode_postprocessing.c
@@ -41,7 +41,8 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
     set_fragment_payload_size(fragment, blocksize);
     set_backend_id(fragment, be->common.id);
     set_backend_version(fragment, be->common.ec_backend_version);
-    
+    set_fragment_adder_size(fragment, be->common.metadata_adder);
+
     if (add_chksum) {
         set_checksum(ct, fragment, blocksize);
     }
diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c
index 7723569..9666b9d 100644
--- a/src/erasurecode_preprocessing.c
+++ b/src/erasurecode_preprocessing.c
@@ -40,16 +40,17 @@ int prepare_fragments_for_encode(ec_backend_t instance,
     int i, ret = 0;
     int data_len;           /* data len to write to fragment headers */
     int aligned_data_len;   /* EC algorithm compatible data length */
-    int bsize = 0;
+    int buffer_size, payload_size = 0;
 
     /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
     data_len = orig_data_size;
     aligned_data_len = get_aligned_data_size(instance, orig_data_size);
-    *blocksize = bsize = (aligned_data_len / k);
+    *blocksize = payload_size = (aligned_data_len / k);
+    buffer_size = payload_size + instance->common.metadata_adder;
 
     for (i = 0; i < k; i++) {
-        int payload_size = data_len > bsize ? bsize : data_len;
-        char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
+        int copy_size = data_len > payload_size ? payload_size : data_len;
+        char *fragment = (char *) alloc_fragment_buffer(buffer_size);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -59,15 +60,15 @@ int prepare_fragments_for_encode(ec_backend_t instance,
         encoded_data[i] = get_data_ptr_from_fragment(fragment);
       
         if (data_len > 0) {
-            memcpy(encoded_data[i], orig_data, payload_size);
+            memcpy(encoded_data[i], orig_data, copy_size);
         }
 
-        orig_data += payload_size;
-        data_len -= payload_size;
+        orig_data += copy_size;
+        data_len -= copy_size;
     }
 
     for (i = 0; i < m; i++) {
-        char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
+        char *fragment = (char *) alloc_fragment_buffer(buffer_size);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -108,7 +109,6 @@ out_error:
  * so in the failure case.
  */
 int prepare_fragments_for_decode(
-        ec_backend_t instance,
         int k, int m,
         char **data, char **parity,
         int  *missing_idxs,
@@ -136,16 +136,14 @@ int prepare_fragments_for_decode(
          * 'data_list'
          */
         if (NULL == data[i]) {
-            data[i] = alloc_fragment_buffer(
-                instance, fragment_size - sizeof(fragment_header_t));
+            data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
             if (NULL == data[i]) {
                 log_error("Could not allocate data buffer!");
                 return -1;
             }
             *realloc_bm = *realloc_bm | (1 << i);
         } else if (!is_addr_aligned((unsigned long)data[i], 16)) {
-            char *tmp_buf = alloc_fragment_buffer(
-                instance, fragment_size - sizeof(fragment_header_t));
+            char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
             if (NULL == tmp_buf) {
                 log_error("Could not allocate temp buffer!");
                 return -1;
@@ -177,16 +175,14 @@ int prepare_fragments_for_decode(
          * DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
          */
         if (NULL == parity[i]) {
-            parity[i] = alloc_fragment_buffer(
-                instance, fragment_size-sizeof(fragment_header_t));
+            parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
             if (NULL == parity[i]) {
                 log_error("Could not allocate parity buffer!");
                 return -1;
             }
             *realloc_bm = *realloc_bm | (1 << (k + i));
         } else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
-            char *tmp_buf = alloc_fragment_buffer(
-                instance, fragment_size-sizeof(fragment_header_t));
+            char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
             if (NULL == tmp_buf) {
                 log_error("Could not allocate temp buffer!");
                 return -1;