diff options
| -rw-r--r-- | include/erasurecode/erasurecode.h | 1 | ||||
| -rw-r--r-- | include/erasurecode/erasurecode_helpers.h | 9 | ||||
| -rw-r--r-- | include/erasurecode/erasurecode_preprocessing.h | 1 | ||||
| -rw-r--r-- | src/erasurecode.c | 4 | ||||
| -rw-r--r-- | src/erasurecode_helpers.c | 52 | ||||
| -rw-r--r-- | src/erasurecode_postprocessing.c | 3 | ||||
| -rw-r--r-- | src/erasurecode_preprocessing.c | 30 | ||||
| -rw-r--r-- | test/liberasurecode_test.c | 5 |
8 files changed, 71 insertions, 34 deletions
diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h index 4bd2a59..f811555 100644 --- a/include/erasurecode/erasurecode.h +++ b/include/erasurecode/erasurecode.h @@ -247,6 +247,7 @@ fragment_metadata { uint32_t idx; /* 4 */ uint32_t size; /* 4 */ + uint32_t frag_adder_size; /* 4 */ uint64_t orig_data_size; /* 8 */ uint8_t chksum_type; /* 1 */ uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */ diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h index 23b3ac8..f7eefd7 100644 --- a/include/erasurecode/erasurecode_helpers.h +++ b/include/erasurecode/erasurecode_helpers.h @@ -45,12 +45,12 @@ typedef struct __attribute__((__packed__)) fragment_header_s { - fragment_metadata_t meta; /* 55 bytes */ + fragment_metadata_t meta; /* 59 bytes */ uint32_t magic; /* 4 bytes */ uint32_t libec_version; /* 4 bytes */ // We must be aligned to 16-byte boundaries // So, size this array accordingly - uint8_t aligned_padding[1]; + uint8_t aligned_padding[13]; } fragment_header_t; /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ @@ -119,7 +119,7 @@ void init_fragment_header(char *buf) void *alloc_zeroed_buffer(int size); void *alloc_and_set_buffer(int size, int value); void *check_and_free_buffer(void *buf); -char *alloc_fragment_buffer(ec_backend_t instance, int size); +char *alloc_fragment_buffer(int size); int free_fragment_buffer(char *buf); void *get_aligned_buffer16(int size); int get_aligned_data_size(ec_backend_t instance, int data_len); @@ -135,6 +135,9 @@ int set_fragment_idx(char *buf, int idx); int get_fragment_idx(char *buf); int set_fragment_payload_size(char *buf, int size); int get_fragment_payload_size(char *buf); +int set_fragment_adder_size(char *buf, int size); +int get_fragment_adder_size(char *buf); +int get_fragment_buffer_size(char *buf); int set_orig_data_size(char *buf, int orig_data_size); int get_orig_data_size(char *buf); int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize); diff --git a/include/erasurecode/erasurecode_preprocessing.h b/include/erasurecode/erasurecode_preprocessing.h index 0b479c6..7ca891d 100644 --- a/include/erasurecode/erasurecode_preprocessing.h +++ b/include/erasurecode/erasurecode_preprocessing.h @@ -37,7 +37,6 @@ int prepare_fragments_for_encode( int *blocksize); int prepare_fragments_for_decode( - ec_backend_t instance, int k, int m, char **data, char **parity, int *missing_idxs, diff --git a/src/erasurecode.c b/src/erasurecode.c index 5bba185..0f626b5 100644 --- a/src/erasurecode.c +++ b/src/erasurecode.c @@ -631,7 +631,7 @@ int liberasurecode_decode(int desc, * (realloc_bm). * */ - ret = prepare_fragments_for_decode(instance, k, m, + ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs, &orig_data_size, &blocksize, fragment_len, &realloc_bm); @@ -799,7 +799,7 @@ int liberasurecode_reconstruct_fragment(int desc, * It passes back a bitmap telling us which buffers need to be freed by * us (realloc_bm). */ - ret = prepare_fragments_for_decode(instance, k, m, data, parity, missing_idxs, + ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs, &orig_data_size, &blocksize, fragment_len, &realloc_bm); if (ret < 0) { diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c index f0f3fe0..05bc838 100644 --- a/src/erasurecode_helpers.c +++ b/src/erasurecode_helpers.c @@ -116,16 +116,11 @@ void * check_and_free_buffer(void * buf) return NULL; } -char *alloc_fragment_buffer(ec_backend_t instance, int size) +char *alloc_fragment_buffer(int size) { char *buf; fragment_header_t *header = NULL; - if (NULL != instance){ - /* Account for any custom metadata the backend wants to add in data_len */ - size += instance->common.metadata_adder; - } - size += sizeof(fragment_header_t); buf = get_aligned_buffer16(size); @@ -168,13 +163,11 @@ int free_fragment_buffer(char *buf) */ uint64_t get_fragment_size(char *buf) { - fragment_header_t *header = NULL; if (NULL == buf) return -1; - header = (fragment_header_t *) buf; - return (header->meta.size + sizeof(fragment_header_t)); + return get_fragment_buffer_size(buf) + sizeof(fragment_header_t); } /** @@ -331,6 +324,47 @@ int get_fragment_payload_size(char *buf) return header->meta.size; } +int set_fragment_adder_size(char *buf, int size) +{ + fragment_header_t *header = (fragment_header_t *) buf; + + assert(NULL != header); + if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { + log_error("Invalid fragment header (set adder size)!"); + return -1; + } + + header->meta.frag_adder_size = size; + + return 0; +} + +int get_fragment_adder_size(char *buf) +{ + fragment_header_t *header = (fragment_header_t *) buf; + + assert(NULL != header); + if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { + log_error("Invalid fragment header (get adder size)!"); + return -1; + } + + return header->meta.frag_adder_size; +} + +int get_fragment_buffer_size(char *buf) +{ + fragment_header_t *header = (fragment_header_t *) buf; + + assert(NULL != header); + if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { + log_error("Invalid fragment header (get size)!"); + return -1; + } + + return header->meta.size + header->meta.frag_adder_size; +} + int set_orig_data_size(char *buf, int orig_data_size) { fragment_header_t *header = (fragment_header_t *) buf; diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c index 9ec68c7..435325e 100644 --- a/src/erasurecode_postprocessing.c +++ b/src/erasurecode_postprocessing.c @@ -41,7 +41,8 @@ void add_fragment_metadata(ec_backend_t be, char *fragment, set_fragment_payload_size(fragment, blocksize); set_backend_id(fragment, be->common.id); set_backend_version(fragment, be->common.ec_backend_version); - + set_fragment_adder_size(fragment, be->common.metadata_adder); + if (add_chksum) { set_checksum(ct, fragment, blocksize); } diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c index 7723569..9666b9d 100644 --- a/src/erasurecode_preprocessing.c +++ b/src/erasurecode_preprocessing.c @@ -40,16 +40,17 @@ int prepare_fragments_for_encode(ec_backend_t instance, int i, ret = 0; int data_len; /* data len to write to fragment headers */ int aligned_data_len; /* EC algorithm compatible data length */ - int bsize = 0; + int buffer_size, payload_size = 0; /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/ data_len = orig_data_size; aligned_data_len = get_aligned_data_size(instance, orig_data_size); - *blocksize = bsize = (aligned_data_len / k); + *blocksize = payload_size = (aligned_data_len / k); + buffer_size = payload_size + instance->common.metadata_adder; for (i = 0; i < k; i++) { - int payload_size = data_len > bsize ? bsize : data_len; - char *fragment = (char *) alloc_fragment_buffer(instance, bsize); + int copy_size = data_len > payload_size ? payload_size : data_len; + char *fragment = (char *) alloc_fragment_buffer(buffer_size); if (NULL == fragment) { ret = -ENOMEM; goto out_error; @@ -59,15 +60,15 @@ int prepare_fragments_for_encode(ec_backend_t instance, encoded_data[i] = get_data_ptr_from_fragment(fragment); if (data_len > 0) { - memcpy(encoded_data[i], orig_data, payload_size); + memcpy(encoded_data[i], orig_data, copy_size); } - orig_data += payload_size; - data_len -= payload_size; + orig_data += copy_size; + data_len -= copy_size; } for (i = 0; i < m; i++) { - char *fragment = (char *) alloc_fragment_buffer(instance, bsize); + char *fragment = (char *) alloc_fragment_buffer(buffer_size); if (NULL == fragment) { ret = -ENOMEM; goto out_error; @@ -108,7 +109,6 @@ out_error: * so in the failure case. */ int prepare_fragments_for_decode( - ec_backend_t instance, int k, int m, char **data, char **parity, int *missing_idxs, @@ -136,16 +136,14 @@ int prepare_fragments_for_decode( * 'data_list' */ if (NULL == data[i]) { - data[i] = alloc_fragment_buffer( - instance, fragment_size - sizeof(fragment_header_t)); + data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t)); if (NULL == data[i]) { log_error("Could not allocate data buffer!"); return -1; } *realloc_bm = *realloc_bm | (1 << i); } else if (!is_addr_aligned((unsigned long)data[i], 16)) { - char *tmp_buf = alloc_fragment_buffer( - instance, fragment_size - sizeof(fragment_header_t)); + char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t)); if (NULL == tmp_buf) { log_error("Could not allocate temp buffer!"); return -1; @@ -177,16 +175,14 @@ int prepare_fragments_for_decode( * DO NOT FREE: the python GC should free the original when cleaning up 'data_list' */ if (NULL == parity[i]) { - parity[i] = alloc_fragment_buffer( - instance, fragment_size-sizeof(fragment_header_t)); + parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t)); if (NULL == parity[i]) { log_error("Could not allocate parity buffer!"); return -1; } *realloc_bm = *realloc_bm | (1 << (k + i)); } else if (!is_addr_aligned((unsigned long)parity[i], 16)) { - char *tmp_buf = alloc_fragment_buffer( - instance, fragment_size-sizeof(fragment_header_t)); + char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t)); if (NULL == tmp_buf) { log_error("Could not allocate temp buffer!"); return -1; diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c index 04d06cf..3d9ef6c 100644 --- a/test/liberasurecode_test.c +++ b/test/liberasurecode_test.c @@ -592,8 +592,11 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id, int num_avail_frags = 0; char *orig_data_ptr = NULL; int remaining = 0; + ec_backend_t be = NULL; desc = liberasurecode_instance_create(be_id, args); + be = liberasurecode_backend_instance_get_by_desc(desc); + if (-EBACKENDNOTAVAIL == desc) { fprintf (stderr, "Backend library not available!\n"); return; @@ -614,7 +617,7 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id, assert(header != NULL); fragment_metadata_t metadata = header->meta; assert(metadata.idx == i); - assert(metadata.size == encoded_fragment_len - frag_header_size); + assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.metadata_adder); assert(metadata.orig_data_size == orig_data_size); char *data_ptr = frag + frag_header_size; int cmp_size = remaining >= metadata.size ? metadata.size : remaining; |