diff options
author | Kevin Greenan <kmgreen2@gmail.com> | 2014-07-30 21:54:07 -0700 |
---|---|---|
committer | Kevin Greenan <kmgreen2@gmail.com> | 2014-08-02 10:11:51 -0700 |
commit | 9d67b8726dfee9615d676847145fee6ed9f2f40d (patch) | |
tree | c77d6f9edd9f22c5d3f8a740487bd001f221390b /src | |
parent | f81a59b59248f96ba2e1b5ab09fdce4cebcf67d9 (diff) | |
download | pyeclib-9d67b8726dfee9615d676847145fee6ed9f2f40d.tar.gz |
Plugging liberasurecode functions into pyeclib
Python changes needed to plug-in liberasurecode.
Changes needed to get PyECLib working with liberasurecode and some of the unit tests passing.
Diffstat (limited to 'src')
-rw-r--r-- | src/c/pyeclib_c/pyeclib_c.c | 1752 | ||||
-rw-r--r-- | src/c/pyeclib_c/pyeclib_c.h | 112 | ||||
-rw-r--r-- | src/python/pyeclib/core.py | 68 | ||||
-rw-r--r-- | src/python/pyeclib/ec_iface.py | 8 |
4 files changed, 209 insertions, 1731 deletions
diff --git a/src/c/pyeclib_c/pyeclib_c.c b/src/c/pyeclib_c/pyeclib_c.c index 6827c09..4a29fd5 100644 --- a/src/c/pyeclib_c/pyeclib_c.c +++ b/src/c/pyeclib_c/pyeclib_c.c @@ -29,13 +29,8 @@ /* Compat layer for python <= 2.6 */ #include "capsulethunk.h" -#include <xor_code.h> -#include <reed_sol.h> -#include <alg_sig.h> -#include <cauchy.h> -#include <jerasure.h> -#include <liberation.h> -#include <galois.h> +#include <erasurecode.h> +#include <erasurecode_helpers.h> #include <math.h> #include <pyeclib_c.h> #include <bytesobject.h> @@ -68,26 +63,6 @@ #endif -/* - * TODO (kmg): Cauchy restriction (k*w*PACKETSIZE) < data_len / k, otherwise you could - * end up with full-blocks of padding - * - */ - -/* - * TODO (kmg): Do a big cleanup: standardize naming, types (typedef stuff), comments, standard error - * handling and refactor where needed. - */ - -/* - * TODO (kmg): Need to clean-up all of the reference counting related stuff. That is, INCREF before - * calling a function that will "steal" a reference and create functions to DECREF stuff. - */ - -/* - * Make the buffer alignment code in 'encode' generic and simple... It sucks right now. - */ - static PyObject *PyECLibError; @@ -98,667 +73,11 @@ static PyObject * pyeclib_c_init(PyObject *self, PyObject *args); static void pyeclib_c_destructor(PyObject *obj); static PyObject * pyeclib_c_get_segment_info(PyObject *self, PyObject *args); static PyObject * pyeclib_c_encode(PyObject *self, PyObject *args); -static PyObject * pyeclib_c_fragments_to_string(PyObject *self, PyObject *args); -static PyObject * pyeclib_c_get_fragment_partition(PyObject *self, PyObject *args); static PyObject * pyeclib_c_reconstruct(PyObject *self, PyObject *args); static PyObject * pyeclib_c_decode(PyObject *self, PyObject *args); static PyObject * pyeclib_c_get_metadata(PyObject *self, PyObject *args); static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args); - -/* - * Determine if an address is aligned to a particular boundary - */ -static int is_addr_aligned(unsigned long addr, int align) -{ - return (addr & (align-1)) == 0; -} - -/* - * Validate the basic erasure code parameters - */ -static int validate_args(int k, int m, int w, pyeclib_type_t type) -{ - - switch (type) { - case PYECC_RS_CAUCHY_ORIG: - { - if (w < 31 && (k+m) > (1 << w)) { - return 0; - } - } - case PYECC_XOR_HD_4: - case PYECC_XOR_HD_3: - return 1; - case PYECC_RS_VAND: - default: - { - long long max_symbols; - - if (w != 8 && w != 16 && w != 32) { - return 0; - } - max_symbols = 1LL << w; - if ((k+m) > max_symbols) { - return 0; - } - } - } - return 1; -} - -/* - * Convert an int list into a bitmap - * Assume the list is '-1' terminated. - */ -static unsigned long long convert_list_to_bitmap(int *list) -{ - int i = 0; - unsigned long long bm = 0; - - while (list[i] > -1) { - /* - * TODO: Assert list[i] < 64 - */ - bm |= (1 << list[i]); - i++; - } - - return bm; -} - -static -void init_fragment_header(char *buf) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - header->magic = PYECC_HEADER_MAGIC; -} - - -/** - * Memory Management Methods - * - * The following methods provide wrappers for allocating and deallocating - * memory. - * - * Future discussions may want to consider moving closer to the recommended - * guidelines in the Python\C API reference manual. One potential issue, - * however, may be how we enforce memory alignment in the Python heap. - * - * 2.7: https://docs.python.org/2.7/c-api/memory.html - * 3.4: https://docs.python.org/3.4/c-api/memory.html - */ - -/** - * Allocate a zero-ed buffer of a specific size. This method allocates from - * the Python stack in order to comply with the recommendations of the - * Python\C API. On error, return NULL and call PyErr_NoMemory. - * - * @param size integer size in bytes of buffer to allocate - * @return pointer to start of allocated buffer or NULL on error - */ -static -void * alloc_zeroed_buffer(int size) -{ - void * buf = NULL; /* buffer to allocate and return */ - - /* Allocate and zero the buffer, or set the appropriate error */ - buf = PyMem_Malloc((size_t) size); - if (buf) { - buf = memset(buf, 0, (size_t) size); - } else { - buf = (void *) PyErr_NoMemory(); - } - - return buf; -} - - -/** - * Deallocate memory buffer if it's not NULL. This methods returns NULL so - * that you can free and reset a buffer using a single line as follows: - * - * my_ptr = check_and_free_buffer(my_ptr); - * - * @return NULL - */ -static -void * check_and_free_buffer(void * buf) -{ - if (buf) PyMem_Free(buf); - - return NULL; -} - - -/** - * Allocates the resources required for the algebraic signatures. On error, - * PyErr_NoMemory is called to set the appropriate error response and NULL - * is returned. - * - * NOTE: It is the caller's responsibility to free the memory allocated here! - * - * @param sig_len integer signature length - * @param gf_w integer galois field size - * @return ptr to allocated and initialized algebraic signature tables - */ -static -alg_sig_t *alloc_alg_sig(int sig_len, int gf_w) -{ - alg_sig_t *buf = NULL; /* algebraic signature handle to allocate */ - - buf = init_alg_sig(sig_len, gf_w); - if (NULL == buf) { - PyErr_NoMemory(); - } - - return buf; -} - - -/** - * Deallocate the resources used for algebraic signatures. - * - * @return NULL - */ -static -void * check_and_free_alg_sig(alg_sig_t *buf) -{ - if (buf) destroy_alg_sig(buf); - - return NULL; -} - - -/** - * Allocate and zero out a buffer aligned to a 16-byte boundary in order - * to support 128-bit operations. On error, call PyErr_NoMemory to set - * the appropriate error string and return NULL. - * - * @param size integer size in bytes of buffer to allocate - * @return pointer to start of allocated buffer, or NULL on error - */ -static -void* alloc_aligned_buffer16(int size) -{ - void *buf = NULL; - - /* - * Ensure all memory is aligned to - * 16-byte boundaries to support - * 128-bit operations - */ - if (posix_memalign(&buf, 16, size) != 0) { - return PyErr_NoMemory(); - } else { - memset(buf, 0, size); - } - - return buf; -} - - -/** - * Allocate an initialized fragment buffer. On error, return NULL and - * preserve call to PyErr_NoMemory. Note, all allocated memory is aligned - * to 16-bytes boundaries in order to support 128-bit operations. - * - * @param size integer size in bytes of buffer to allocate - * @return pointer to start of allocated fragment or NULL on error - */ -static -char *alloc_fragment_buffer(int size) -{ - char *buf = NULL; - fragment_header_t *header = NULL; - - /* Calculate size needed for fragment header + data */ - size += sizeof(fragment_header_t); - - /* Allocate and init the aligned buffer, or set the appropriate error */ - buf = alloc_aligned_buffer16(size); - if (buf) { - init_fragment_header(buf); - } - - return buf; -} - - -/** - * Deallocate a fragment buffer. This method confirms via magic number that - * the passed in buffer is a proper fragment buffer. - * - * @param buf pointer to fragment buffer - * @return 0 on successful free, -1 on error - */ -static -int free_fragment_buffer(char *buf) -{ - fragment_header_t *header; - if (buf == NULL) { - return -1; - } - - buf -= sizeof(fragment_header_t); - - header = (fragment_header_t*)buf; - if (header->magic != PYECC_HEADER_MAGIC) { - PyErr_SetString(PyECLibError, "Invalid fragment header (free fragment)!"); - return -1; - } - free(buf); - - return 0; -} - - -/** - * Allocate a zero-ed Python string of a specific size. On error, call - * PyErr_NoMemory and return NULL. - * - * @param size integer size in bytes of zero string to create - * @return pointer to start of allocated zero string or NULL on error - */ -static -PyObject * alloc_zero_string(int size) -{ - char *tmp_data = NULL; /* tmp buffer used in PyObject creation */ - PyObject *zero_string = NULL; /* zero string to return */ - - /* Create the zero-out c-string buffer */ - tmp_data = (char *) alloc_zeroed_buffer(size); - if (NULL == tmp_data) { - return PyErr_NoMemory(); - } - - /* Create the python value to return */ - zero_string = PY_BUILDVALUE_OBJ_LEN(tmp_data, size); - check_and_free_buffer(tmp_data); - - return zero_string; -} - - -static -char* get_data_ptr_from_fragment(char *buf) -{ - char * data_ptr = NULL; - - if (NULL != buf) { - data_ptr = buf + sizeof(fragment_header_t); - } - - return data_ptr; -} - -static -char* get_fragment_ptr_from_data_novalidate(char *buf) -{ - buf -= sizeof(fragment_header_t); - - return buf; -} - -static -int use_inline_chksum(pyeclib_t *pyeclib_handle) -{ - return pyeclib_handle->inline_chksum; -} - -static -int supports_alg_sig(pyeclib_t *pyeclib_handle) -{ - if (!pyeclib_handle->algsig_chksum) return 0; - if (pyeclib_handle->alg_sig_desc != NULL) return 1; - return 0; -} - -static -char* get_fragment_ptr_from_data(char *buf) -{ - fragment_header_t *header; - - buf -= sizeof(fragment_header_t); - - header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - fprintf(stderr, "Invalid fragment header (get header ptr)!\n"); - return NULL; - } - - return buf; -} - -static -int set_chksum(char *buf, int chksum) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - fprintf(stderr, "Invalid fragment header (set chksum)!\n"); - return -1; - } - - header->chksum = chksum; - - return 0; -} - -static -int get_chksum(char *buf) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - PyErr_SetString(PyECLibError, "Invalid fragment header (get chksum)!"); - return -1; - } - - return header->chksum; -} - -static -int set_fragment_idx(char *buf, int idx) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - fprintf(stderr, "Invalid fragment header (idx check)!\n"); - return -1; - } - - header->idx = idx; - - return 0; -} - -static -int get_fragment_idx(char *buf) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - PyErr_SetString(PyECLibError, "Invalid fragment header (get idx)!"); - return -1; - } - - return header->idx; -} - -static -int set_fragment_size(char *buf, int size) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - PyErr_SetString(PyECLibError, "Invalid fragment header (size check)!"); - return -1; - } - - header->size = size; - - return 0; -} - -static -int get_fragment_size(char *buf) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - PyErr_SetString(PyECLibError, "Invalid fragment header (get size)!"); - return -1; - } - - return header->size; -} - -static -int set_orig_data_size(char *buf, int orig_data_size) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - PyErr_SetString(PyECLibError, "Invalid fragment header (set orig data check)!"); - return -1; - } - - header->orig_data_size = orig_data_size; - - return 0; -} - -static -int get_orig_data_size(char *buf) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - PyErr_SetString(PyECLibError, "Invalid fragment header (get orig data check)!"); - return -1; - } - - return header->orig_data_size; -} - -static -int validate_fragment(char *buf) -{ - fragment_header_t* header = (fragment_header_t*)buf; - - if (header->magic != PYECC_HEADER_MAGIC) { - return -1; - } - - return 0; -} - - -/* - * Prepares the fragments and helper data structures for decoding. Note, - * buffers for data, parity and missing_idxs must be allocated by the caller. - * - * @param *pyeclib_handle - * @param *data_list python list of data fragments - * @param *parity_list python list of parity fragments - * @param *missing_idx_list python list of indexes of missing fragments - * @param **data allocated k-length array of data buffers - * @param **parity allocated m-length array of parity buffers - * @param *missing_idxs array of missing indexes with extra space for -1 terminator - * @param *orig_size original size of object from the fragment header - * @param fragment_size integer size in bytes of fragment - * @return 0 on success, -1 on error - */ -static int get_decoding_info(pyeclib_t *pyeclib_handle, - PyObject *data_list, - PyObject *parity_list, - PyObject *missing_idx_list, - char **data, - char **parity, - int *missing_idxs, - int *orig_size, - int fragment_size, - unsigned long long *realloc_bm) -{ - int i; /* a counter */ - int data_size = 0; /* number of data fragments provided */ - int parity_size = 0; /* number of parity fragments provided */ - int missing_size = 0; /* number of missing index entries */ - int orig_data_size = -1; /* data size (B) from fragment header */ - unsigned long long missing_bm; /* bitmap form of missing indexes list */ - int needs_addr_alignment = PYECLIB_NEEDS_ADDR_ALIGNMENT(pyeclib_handle->type); - - /* Validate the list sizes */ - data_size = (int)PyList_Size(data_list); - parity_size = (int)PyList_Size(parity_list); - missing_size = (int)PyList_Size(missing_idx_list); - if (data_size != pyeclib_handle->k) { - return -1; - } - if (parity_size != pyeclib_handle->m) { - return -1; - } - - /* Record missing indexes in a -1 terminated array and bitmap */ - for (i = 0; i < missing_size; i++) { - PyObject *obj_idx = PyList_GetItem(missing_idx_list, i); - - missing_idxs[i] = (int) PyLong_AsLong(obj_idx);; - } - missing_idxs[i] = -1; - missing_bm = convert_list_to_bitmap(missing_idxs); - - /* - * Determine if each data fragment is: - * 1.) Alloc'd: if not, alloc new buffer (for missing fragments) - * 2.) Aligned to 16-byte boundaries: if not, alloc a new buffer - * memcpy the contents and free the old buffer - */ - for (i = 0; i < pyeclib_handle->k; i++) { - PyObject *tmp_data = PyList_GetItem(data_list, i); - Py_ssize_t len = 0; - PyBytes_AsStringAndSize(tmp_data, &(data[i]), &len); - - /* - * Allocate or replace with aligned buffer if the buffer was not aligned. - * DO NOT FREE: the python GC should free the original when cleaning up 'data_list' - */ - if (len == 0 || !data[i]) { - data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t)); - if (NULL == data[i]) { - return -1; - } - *realloc_bm = *realloc_bm | (1 << i); - } else if ((needs_addr_alignment && !is_addr_aligned((unsigned long)data[i], 16))) { - char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t)); - memcpy(tmp_buf, data[i], fragment_size); - data[i] = tmp_buf; - *realloc_bm = *realloc_bm | (1 << i); - } - - /* Need to determine the size of the original data */ - if (((missing_bm & (1 << i)) == 0) && orig_data_size < 0) { - orig_data_size = get_orig_data_size(data[i]); - if (orig_data_size < 0) { - return -1; - } - } - - /* Set the data element to the fragment payload */ - data[i] = get_data_ptr_from_fragment(data[i]); - if (data[i] == NULL) { - return -1; - } - } - - /* Perform the same allocation, alignment checks on the parity fragments */ - for (i=0; i < pyeclib_handle->m; i++) { - PyObject *tmp_parity = PyList_GetItem(parity_list, i); - Py_ssize_t len = 0; - PyBytes_AsStringAndSize(tmp_parity, &(parity[i]), &len); - - /* - * Allocate or replace with aligned buffer, if the buffer was not aligned. - * DO NOT FREE: the python GC should free the original when cleaning up 'data_list' - */ - if (len == 0 || !parity[i]) { - parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t)); - if (NULL == parity[i]) { - return -1; - } - *realloc_bm = *realloc_bm | (1 << (pyeclib_handle->k + i)); - } else if (needs_addr_alignment && !is_addr_aligned((unsigned long)parity[i], 16)) { - char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t)); - memcpy(tmp_buf, parity[i], fragment_size); - parity[i] = tmp_buf; - *realloc_bm = *realloc_bm | (1 << (pyeclib_handle->k + i)); - } - - /* Set the parity element to the fragment payload */ - parity[i] = get_data_ptr_from_fragment(parity[i]); - if (parity[i] == NULL) { - return -1; - } - } - - *orig_size = orig_data_size; - return 0; -} - - -/** - * Compute a size aligned to the number of data and the underlying wordsize - * of the EC algorithm. - * - * @param pyeclib_handle eclib object with EC configurations - * @param data_len integer length of data in bytes - * @return integer data length aligned with wordsize of EC algorithm - */ -static int -get_aligned_data_size(pyeclib_t* pyeclib_handle, int data_len) -{ - int word_size = pyeclib_handle->w / 8; - int alignment_multiple; - int aligned_size = 0; - - /* - * For Cauchy reed-solomon align to k*word_size*packet_size - * For Vandermonde reed-solomon and flat-XOR, align to k*word_size - */ - if (pyeclib_handle->type == PYECC_RS_CAUCHY_ORIG) { - alignment_multiple = pyeclib_handle->k * pyeclib_handle->w * PYECC_CAUCHY_PACKETSIZE; - } else { - alignment_multiple = pyeclib_handle->k * word_size; - } - - aligned_size = (int)ceill((double)data_len / alignment_multiple) * alignment_multiple; - - return aligned_size; -} - - -static -int get_minimum_encode_size(pyeclib_t *pyeclib_handle) -{ - return get_aligned_data_size(pyeclib_handle, 1); -} - - -static -int get_fragment_metadata(pyeclib_t *pyeclib_handle, char *fragment_buf, fragment_metadata_t *fragment_metadata) -{ - char *fragment_data = get_data_ptr_from_fragment(fragment_buf); - int fragment_size = get_fragment_size(fragment_buf); - int fragment_idx = get_fragment_idx(fragment_buf); - - memset(fragment_metadata, 0, sizeof(fragment_metadata_t)); - - fragment_metadata->size = fragment_size; - fragment_metadata->idx = fragment_idx; - - /* - * If w \in [8, 16] and using RS_VAND or XOR_CODE, then use Alg_sig - * Else use CRC32 - */ - if (supports_alg_sig(pyeclib_handle)) { - // Compute algebraic signature - compute_alg_sig(pyeclib_handle->alg_sig_desc, fragment_data, fragment_size, fragment_metadata->signature); - } else if (use_inline_chksum(pyeclib_handle)) { - int stored_chksum = get_chksum(fragment_buf); - int computed_chksum = crc32(0, fragment_data, fragment_size); - - if (stored_chksum != computed_chksum) { - fragment_metadata->chksum_mismatch = 1; - } - } - - return 0; -} - - /** * Constructor method for creating a new pyeclib object using the given parameters. * @@ -773,78 +92,32 @@ pyeclib_c_init(PyObject *self, PyObject *args) { pyeclib_t *pyeclib_handle = NULL; PyObject *pyeclib_obj_handle = NULL; - int k, m, w; + int k, m, hd=0; int use_inline_chksum = 0, use_algsig_chksum = 0; const char *type_str; - pyeclib_type_t type; /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "iis|ii", &k, &m, &type_str, &use_inline_chksum, &use_algsig_chksum)) { + if (!PyArg_ParseTuple(args, "iis|iii", &k, &m, &type_str, &hd, &use_inline_chksum, &use_algsig_chksum)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.init"); return NULL; } - type = get_ecc_type(type_str); - if (type == PYECC_NOT_FOUND) { - PyErr_SetString(PyECLibError, "Invalid type passed to pyeclib.init"); - return NULL; - } - - w = get_best_w_for_ecc_type(type); - if (w < 0) { - /* this should not happen */ - PyErr_SetString(PyECLibError, "Invalid w value. pyeclib internal error"); - return NULL; - } - - if (!validate_args(k, m, w, type)) { - PyErr_SetString(PyECLibError, "Invalid args passed to pyeclib.init"); - return NULL; - } /* Allocate and initialize the pyeclib object */ pyeclib_handle = (pyeclib_t *) alloc_zeroed_buffer(sizeof(pyeclib_t)); if (NULL == pyeclib_handle) { goto error; } - pyeclib_handle->k = k; - pyeclib_handle->m = m; - pyeclib_handle->w = w; - pyeclib_handle->type = type; - pyeclib_handle->inline_chksum = use_inline_chksum; - pyeclib_handle->algsig_chksum = use_algsig_chksum; - pyeclib_handle->alg_sig_desc = NULL; - - /* Allocate and initialize additional resources */ - switch (type) { - case PYECC_RS_CAUCHY_ORIG: - pyeclib_handle->matrix = cauchy_original_coding_matrix(k, m, w); - pyeclib_handle->bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, pyeclib_handle->matrix); - pyeclib_handle->schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, pyeclib_handle->bitmatrix); - break; - case PYECC_XOR_HD_3: - pyeclib_handle->xor_code_desc = init_xor_hd_code(k, m, 3); - if (pyeclib_handle->algsig_chksum) { - pyeclib_handle->alg_sig_desc = alloc_alg_sig(32, 16); - if (NULL == pyeclib_handle->alg_sig_desc) goto error; - } - break; - case PYECC_XOR_HD_4: - pyeclib_handle->xor_code_desc = init_xor_hd_code(k, m, 4); - if (pyeclib_handle->algsig_chksum) { - pyeclib_handle->alg_sig_desc = alloc_alg_sig(32, 16); - if (NULL == pyeclib_handle->alg_sig_desc) goto error; - } - break; - case PYECC_RS_VAND: - default: - pyeclib_handle->matrix = reed_sol_vandermonde_coding_matrix(k, m, w); - if (pyeclib_handle->algsig_chksum && w == 8) { - pyeclib_handle->alg_sig_desc = alloc_alg_sig(32, 8); - if (NULL == pyeclib_handle->alg_sig_desc) goto error; - } else if (pyeclib_handle->algsig_chksum && w == 16) { - pyeclib_handle->alg_sig_desc = alloc_alg_sig(32, 16); - if (NULL == pyeclib_handle->alg_sig_desc) goto error; - } + + pyeclib_handle->ec_args.k = k; + pyeclib_handle->ec_args.m = m; + pyeclib_handle->ec_args.hd = hd; + pyeclib_handle->ec_args.inline_chksum = use_inline_chksum; + pyeclib_handle->ec_args.algsig_chksum = use_algsig_chksum; + + pyeclib_handle->ec_desc = liberasurecode_instance_create(type_str, &(pyeclib_handle->ec_args)); + if (pyeclib_handle->ec_desc <= 0) { + PyErr_SetString(PyECLibError, "Invalid arguments passed to liberasurecode_instance_create"); + goto error; } /* Prepare the python object to return */ @@ -868,9 +141,6 @@ pyeclib_c_init(PyObject *self, PyObject *args) goto exit; error: - if (NULL != pyeclib_handle) { - check_and_free_alg_sig(pyeclib_handle->alg_sig_desc); - } check_and_free_buffer(pyeclib_handle); pyeclib_obj_handle = NULL; @@ -896,10 +166,8 @@ pyeclib_c_destructor(PyObject *obj) if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Attempted to free an invalid reference to pyeclib_handle"); } else { - check_and_free_alg_sig(pyeclib_handle->alg_sig_desc); check_and_free_buffer(pyeclib_handle); } - return; } @@ -959,7 +227,7 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args) } /* The minimum segment size depends on the EC algorithm */ - min_segment_size = get_minimum_encode_size(pyeclib_handle); + min_segment_size = liberasurecode_get_minimum_encode_size(pyeclib_handle->ec_desc); /* Get the number of segments */ num_segments = (int)ceill((double)data_len / segment_size); @@ -983,10 +251,10 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args) * This will copmpute a size algined to the number of data * and the underlying wordsize of the EC algorithm. */ - aligned_data_len = get_aligned_data_size(pyeclib_handle, data_len); + aligned_data_len = liberasurecode_get_aligned_data_size(pyeclib_handle->ec_desc, data_len); /* aligned_data_len is guaranteed to be divisible by k */ - fragment_size = aligned_data_len / pyeclib_handle->k; + fragment_size = aligned_data_len / pyeclib_handle->ec_args.k; /* Segment size is the user-provided segment size */ segment_size = data_len; @@ -998,10 +266,10 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args) * the minimum segment size. */ - aligned_segment_size = get_aligned_data_size(pyeclib_handle, segment_size); + aligned_segment_size = liberasurecode_get_aligned_data_size(pyeclib_handle->ec_desc, segment_size); /* aligned_data_len is guaranteed to be divisible by k */ - fragment_size = aligned_segment_size / pyeclib_handle->k; + fragment_size = aligned_segment_size / pyeclib_handle->ec_args.k; last_segment_size = data_len - (segment_size * (num_segments - 1)); @@ -1017,10 +285,10 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args) last_segment_size = last_segment_size + segment_size; } - aligned_segment_size = get_aligned_data_size(pyeclib_handle, last_segment_size); + aligned_segment_size = liberasurecode_get_aligned_data_size(pyeclib_handle->ec_desc, last_segment_size); /* Compute the last fragment size from the last segment size */ - last_fragment_size = aligned_segment_size / pyeclib_handle->k; + last_fragment_size = aligned_segment_size / pyeclib_handle->ec_args.k; } /* Add header to fragment sizes */ @@ -1055,14 +323,12 @@ pyeclib_c_encode(PyObject *self, PyObject *args) { PyObject *pyeclib_obj_handle = NULL; pyeclib_t *pyeclib_handle= NULL; - char **data_to_encode = NULL; /* array of k data buffers */ + char **encoded_data = NULL; /* array of k data buffers */ char **encoded_parity = NULL; /* array of m parity buffers */ PyObject *list_of_strips = NULL; /* list of encoded strips to return */ char *data; /* param, data buffer to encode */ int data_len; /* param, length of data buffer */ - int aligned_data_len; /* EC algorithm compatible data length */ - int orig_data_size; /* data length to write to headers */ - int blocksize; /* length of each of k data elements */ + uint64_t fragment_len; /* length, in bytes of the fragments */ int i; /* a counter */ /* Assume binary data (force "byte array" input) */ @@ -1075,428 +341,37 @@ pyeclib_c_encode(PyObject *self, PyObject *args) PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); return NULL; } - - /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/ - orig_data_size = data_len; - aligned_data_len = get_aligned_data_size(pyeclib_handle, data_len); - blocksize = aligned_data_len / pyeclib_handle->k; - - /* Allocate and initialize an array of zero'd out data buffers */ - data_to_encode = (char**) alloc_zeroed_buffer(sizeof(char*) * pyeclib_handle->k); - if (NULL == data_to_encode) { - goto error; - } - for (i = 0; i < pyeclib_handle->k; i++) { - int payload_size = data_len > blocksize ? blocksize : data_len; - char *fragment = alloc_fragment_buffer(blocksize); - if (NULL == fragment) { - goto error; - } - - /* Copy existing data into clean, zero'd out buffer */ - data_to_encode[i] = get_data_ptr_from_fragment(fragment); - if (data_len > 0) { - memcpy(data_to_encode[i], data, payload_size); - } - /* Fragment size will always be the same (may be able to get rid of this) */ - set_fragment_size(fragment, blocksize); + if (liberasurecode_encode(pyeclib_handle->ec_desc, + data, data_len, &encoded_data, &encoded_parity, &fragment_len) < 0) { - data += payload_size; - data_len -= payload_size; - } - - /* Allocate and initialize an array of zero'd out parity buffers */ - encoded_parity = (char**) alloc_zeroed_buffer(sizeof(char*) * pyeclib_handle->m); - if (NULL == encoded_parity) { - goto error; - } - for (i = 0; i < pyeclib_handle->m; i++) { - char *fragment = alloc_fragment_buffer(blocksize); - if (NULL == fragment) { - goto error; - } - - encoded_parity[i] = get_data_ptr_from_fragment(fragment); - set_fragment_size(fragment, blocksize); - } - - /* Run the erasure coding algorithm to generate the parity fragments */ - switch (pyeclib_handle->type) { - case PYECC_RS_CAUCHY_ORIG: - jerasure_bitmatrix_encode(pyeclib_handle->k, pyeclib_handle->m, - pyeclib_handle->w, pyeclib_handle->bitmatrix, - data_to_encode, encoded_parity, blocksize, - PYECC_CAUCHY_PACKETSIZE); - break; - case PYECC_XOR_HD_3: - case PYECC_XOR_HD_4: - pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc, - data_to_encode, encoded_parity, - blocksize); - break; - case PYECC_RS_VAND: - default: - jerasure_matrix_encode(pyeclib_handle->k, pyeclib_handle->m, - pyeclib_handle->w, pyeclib_handle->matrix, - data_to_encode, encoded_parity, blocksize); - break; + PyErr_SetString(PyECLibError, "Encountered error in liberasurecode_encode"); + return NULL; } - + /* Create the python list of fragments to return */ - list_of_strips = PyList_New(pyeclib_handle->k + pyeclib_handle->m); + list_of_strips = PyList_New(pyeclib_handle->ec_args.k + pyeclib_handle->ec_args.m); if (NULL == list_of_strips) { PyErr_SetString(PyECLibError, "Error allocating python list in encode"); - goto error; + return NULL; } - /* Finalize data fragments and add them to the python list to return */ - for (i = 0; i < pyeclib_handle->k; i++) { - char *fragment_ptr = get_fragment_ptr_from_data(data_to_encode[i]); - int fragment_size = blocksize + sizeof(fragment_header_t); - set_fragment_idx(fragment_ptr, i); - set_orig_data_size(fragment_ptr, orig_data_size); - if (use_inline_chksum(pyeclib_handle)) { - int chksum = crc32(0, data_to_encode[i], blocksize); - set_chksum(fragment_ptr, chksum); - } + /* Add data fragments to the python list to return */ + for (i = 0; i < pyeclib_handle->ec_args.k; i++) { PyList_SET_ITEM(list_of_strips, i, - PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); + PY_BUILDVALUE_OBJ_LEN(encoded_data[i], fragment_len)); } - /* Finalize parity fragments and add them to the python list to return */ - for (i = 0; i < pyeclib_handle->m; i++) { - char *fragment_ptr = get_fragment_ptr_from_data(encoded_parity[i]); - int fragment_size = blocksize + sizeof(fragment_header_t); - set_fragment_idx(fragment_ptr, pyeclib_handle->k+i); - set_orig_data_size(fragment_ptr, orig_data_size); - if (use_inline_chksum(pyeclib_handle)) { - int chksum = crc32(0, encoded_parity[i], blocksize); - set_chksum(fragment_ptr, chksum); - } - PyList_SET_ITEM(list_of_strips, pyeclib_handle->k + i, - PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); - } - - goto exit; - -error: - list_of_strips = NULL; - -exit: - if (data_to_encode) { - for (i = 0; i < pyeclib_handle->k; i++) { - if (data_to_encode[i]) free_fragment_buffer(data_to_encode[i]); - } - check_and_free_buffer(data_to_encode); - } - if (encoded_parity) { - for (i = 0; i < pyeclib_handle->m; i++) { - if (encoded_parity[i]) free_fragment_buffer(encoded_parity[i]); - } - check_and_free_buffer(encoded_parity); + /* Add parity fragments to the python list to return */ + for (i = 0; i < pyeclib_handle->ec_args.m; i++) { + PyList_SET_ITEM(list_of_strips, pyeclib_handle->ec_args.k + i, + PY_BUILDVALUE_OBJ_LEN(encoded_parity[i], fragment_len)); } return list_of_strips; } -/* - * Convert a set of fragments into a string. If, less than k data fragments - * are present, return None. Return NULL on error. - * - * @param pyeclib_obj_handle - * @param list of fragments - * @param python string or None, NULL on error - */ -static PyObject * -pyeclib_c_fragments_to_string(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t *pyeclib_handle = NULL; - PyObject *fragment_list = NULL; /* param, python list of fragments */ - PyObject *ret_string = NULL; /* python string to return */ - char *ret_cstring = NULL; /* c string to build return string from */ - int ret_data_size; /* size of return string in bytes */ - char **data = NULL; /* array of data buffers */ - int string_off = 0; /* offset into cstring */ - int num_fragments = 0; /* num of fragments provided by caller */ - int num_data = 0; /* num of fragments that are data */ - int orig_data_size = -1; /* data size from fragment header */ - int i = 0; /* a counter */ - - /* Collect and validate the method arguments */ - if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_list)) { - PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.fragments_to_string"); - return NULL; - } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.fragments_to_string"); - return NULL; - } - if (!PyList_Check(fragment_list)) { - PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list in pyeclib.fragments_to_string"); - return NULL; - } - - /* Return None if there's insufficient fragments */ - num_fragments = (int) PyList_Size(fragment_list); - if (pyeclib_handle->k > num_fragments) { - return Py_BuildValue(""); - } - - /* - * NOTE: Update to only copy original size out of the buffers - */ - - /* - * Iterate over the fragments. If we have all k data fragments, then we can - * concatenate them into a string and return it; otherwise, we return NULL - */ - data = (char **) alloc_zeroed_buffer(sizeof(char *) * pyeclib_handle->k); - if (NULL == data) { - return NULL; - } - for (i = 0; i < num_fragments && num_data < pyeclib_handle->k; i++) { - PyObject *tmp_data = PyList_GetItem(fragment_list, i); - char *tmp_buf; - int index; - int data_size; - Py_ssize_t len; - - /* Get and validate the fragment index and size */ - PyBytes_AsStringAndSize(tmp_data, &tmp_buf, &len); - index = get_fragment_idx(tmp_buf); - data_size = get_fragment_size(tmp_buf); - if ((index < 0) || (data_size < 0)) { - ret_string = NULL; - goto exit; - } - - /* Validate the original data size */ - if (orig_data_size < 0) { - orig_data_size = get_orig_data_size(tmp_buf); - } else { - if (get_orig_data_size(tmp_buf) != orig_data_size) { - PyErr_SetString(PyECLibError, "Inconsistent orig data sizes found in headers"); - ret_string = NULL; - goto exit; - } - } - - /* Skip parity fragments, put data fragments in index order */ - if (index >= pyeclib_handle->k) { - continue; - } else { - data[index] = tmp_buf; - num_data++; - } - } - - /* Return None if there still isn't insufficient fragments */ - if (num_data != pyeclib_handle->k) { - ret_string = Py_BuildValue(""); - goto exit; - } - - /* Create the c-string to return */ - ret_cstring = (char *) alloc_zeroed_buffer(orig_data_size); - if (NULL == ret_cstring) { - ret_string = NULL; - goto exit; - } - ret_data_size = orig_data_size; - - /* Copy fragment data into cstring (fragments should be in index order) */ - for (i = 0; i < num_data && orig_data_size > 0; i++) { - char* fragment_data = get_data_ptr_from_fragment(data[i]); - int fragment_size = get_fragment_size(data[i]); - int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size; - - memcpy(ret_cstring + string_off, fragment_data, payload_size); - orig_data_size -= payload_size; - string_off += payload_size; - } - ret_string = PY_BUILDVALUE_OBJ_LEN(ret_cstring, ret_data_size); - -exit: - check_and_free_buffer(data); - check_and_free_buffer(ret_cstring); - - return ret_string; -} - - -/* - * Return a tuple containing a reference to a data fragment list, a parity - * fragment list and a missing fragment index list, including all '0' - * fragment for missing fragments. Copy references to the existing - * fragments and create new string for missing fragments. - * - * @param pyeclib_obj_handle - * @param list of fragments - * @return (data fragments, parity fragments, missing indexes) tuple - */ -static PyObject * -pyeclib_c_get_fragment_partition(PyObject *self, PyObject *args) -{ - PyObject *pyeclib_obj_handle = NULL; - pyeclib_t* pyeclib_handle = NULL; - PyObject *fragment_list = NULL; /* param, list of fragments */ - PyObject *data_list = NULL; /* data fragments for return tuple */ - PyObject *parity_list = NULL; /* parity fragments for return tuple */ - PyObject *fragment_string = NULL; /* fragment string for lists */ - PyObject *missing_list = NULL; /* missing indexes for return tuple */ - PyObject *return_lists = NULL; /* Python tuple to return */ - PyObject **data = NULL; /* array of k data fragment buffers*/ - PyObject **parity = NULL; /* array of m parity fragment buffers */ - int *missing = NULL; /* indicies of missing fragments */ - int num_missing = 0; /* num of missing fragments */ - int num_fragments; /* num of frags provided by caller */ - int fragment_size = 0; /* size in bytes of fragments */ - int i = 0; /* a counter */ - - /* Collect and validate the method arguments */ - if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_list)) { - PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_fragment_partition"); - return NULL; - } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); - if (pyeclib_handle == NULL) { - PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_fragment_partition"); - return NULL; - } - if (!PyList_Check(fragment_list)) { - PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list in pyeclib.get_fragment_partition"); - return NULL; - } - - /* Create the arrays need to hold the data and parity fragments */ - data = (PyObject **) alloc_zeroed_buffer(pyeclib_handle->k * sizeof(PyObject*)); - if (NULL == data) { - goto error; - } - parity = (PyObject **) alloc_zeroed_buffer(pyeclib_handle->m * sizeof(PyObject*)); - if (NULL == parity) { - goto error; - } - - /* Fill in the data and parity pointers to reveal missing fragments */ - num_fragments = (int) PyList_Size(fragment_list); - for (i = 0; i < num_fragments; i++) { - PyObject *tmp_data = PyList_GetItem(fragment_list, i); - char *c_buf; - int index; - Py_ssize_t len; - - /* ASSUMPTION: The fragment_size is the max of the fragments we read */ - PyBytes_AsStringAndSize(tmp_data, &c_buf, &len); - if (len > fragment_size) { - fragment_size = (int) len; - } - - /* Get fragment index and set the index in data or parity */ - index = get_fragment_idx(c_buf); - if (index < pyeclib_handle->k) { - data[index] = tmp_data; - } else { - parity[index - pyeclib_handle->k] = tmp_data; - } - } - - /* If there are missing bufs, figure out which indexes are missing - * - * ASSUMPTION: We will never try to do anything when we have more - * than k missing fragments - */ - missing = (int *) alloc_zeroed_buffer(pyeclib_handle->k * sizeof(int)); - if (NULL == missing) { - goto error; - } else { - num_missing = 0; - for (i = 0; i < pyeclib_handle->k; i++) { - if (data[i] == NULL) { - missing[num_missing] = i; - num_missing++; - } - } - for (i = 0; i < pyeclib_handle->m; i++) { - if (parity[i] == NULL) { - missing[num_missing] = i + pyeclib_handle->k; - num_missing++; - } - } - } - - /* Create the python objects to return */ - data_list = PyList_New(pyeclib_handle->k); - parity_list = PyList_New(pyeclib_handle->m); - missing_list = PyList_New(num_missing); - return_lists = PyTuple_New(3); - if (!data_list || !parity_list || !missing_list || !return_lists) { - return_lists = PyErr_NoMemory(); - goto exit; - } - - /* Fill in the data fragments, create zero fragment if missing */ - for (i = 0; i < pyeclib_handle->k; i++) { - if (data[i] != NULL) { - /* BORROWED REF: increment ref count before inserting into list */ - Py_INCREF(data[i]); - fragment_string = data[i]; - } else { - fragment_string = alloc_zero_string(fragment_size); - if (NULL == fragment_string) { - goto error; - } - } - PyList_SET_ITEM(data_list, i, fragment_string); - } - - /* Fill in the parity fragments, create zero fragment if missing */ - for (i = 0; i < pyeclib_handle->m; i++) { - if (parity[i] != NULL) { - /* BORROWED REF: increment ref count before inserting into list */ - Py_INCREF(parity[i]); - fragment_string = parity[i]; - } else { - fragment_string = alloc_zero_string(fragment_size); - if (NULL == fragment_string) { - goto error; - } - } - PyList_SET_ITEM(parity_list, i, fragment_string); - } - - /* Fill in the list of missing indexes */ - for (i = 0;i < num_missing; i++) { - PyList_SET_ITEM(missing_list, i, Py_BuildValue("i", missing[i])); - } - - Py_INCREF(data_list); - Py_INCREF(parity_list); - Py_INCREF(missing_list); - - PyTuple_SetItem(return_lists, 0, data_list); - PyTuple_SetItem(return_lists, 1, parity_list); - PyTuple_SetItem(return_lists, 2, missing_list); - //Py_INCREF(return_lists); - - goto exit; - -error: - return_lists = NULL; - -exit: - check_and_free_buffer(data); - check_and_free_buffer(parity); - check_and_free_buffer(missing); - - return return_lists; -} - - /** * Return a list of lists with valid rebuild indexes given an EC algorithm * and a list of missing indexes. @@ -1514,9 +389,8 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args) PyObject *fragment_idx_list = NULL; /* list of req'd indexes to return */ int *c_missing_list = NULL; /* c-array of missing indexes */ int num_missing; /* size of passed in missing list */ - int i = 0, j = 0; /* counters */ + int i = 0; /* counters */ int k, m; /* EC algorithm parameters */ - unsigned long missing_bm = 0; /* bitmap of missing indexes */ int *fragments_needed = NULL; /* indexes of xor code fragments */ int ret; /* return value for xor code */ @@ -1530,8 +404,8 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args) PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_required_fragments"); return NULL; } - k = pyeclib_handle->k; - m = pyeclib_handle->m; + k = pyeclib_handle->ec_args.k; + m = pyeclib_handle->ec_args.m; /* Generate -1 terminated c-array and bitmap of missing indexes */ num_missing = (int) PyList_Size(missing_list); @@ -1545,62 +419,27 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args) long idx = PyLong_AsLong(obj_idx); c_missing_list[i] = (int) idx; } - missing_bm = convert_list_to_bitmap(c_missing_list); + + fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m)); + if (NULL == fragments_needed) { + goto exit; + } + + ret = liberasurecode_fragments_needed(pyeclib_handle->ec_desc, c_missing_list, fragments_needed); + if (ret < 0) { + PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments!"); + goto exit; + } - /* Generate the python list of lists of rebuild indexes to return */ + /* Post-process into a Python list */ fragment_idx_list = PyList_New(0); if (NULL == fragment_idx_list) { goto exit; } - j = 0; - switch(pyeclib_handle->type) { - case PYECC_RS_CAUCHY_ORIG: - case PYECC_RS_VAND: - for (i = 0; i < (k + m); i++) { - if (!(missing_bm & (1 << i))) { - PyList_Append(fragment_idx_list, Py_BuildValue("i", i)); - j++; - } - if (j == k) { - break; - } - } - - if (j != k) { - /* Let the garbage collector clean this up */ - Py_DECREF(fragment_idx_list); - PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments (need at least %d, %d are given)", k, j); - fragment_idx_list = NULL; - } - break; - case PYECC_XOR_HD_3: - case PYECC_XOR_HD_4: - { - fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m)); - if (NULL == fragments_needed) { - fragment_idx_list = NULL; - goto exit; - } - ret = pyeclib_handle->xor_code_desc->fragments_needed(pyeclib_handle->xor_code_desc, - c_missing_list, - fragments_needed); - - if (ret < 0) { - Py_DECREF(fragment_idx_list); - PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments!"); - fragment_idx_list = NULL; - break; - } - - while (fragments_needed[j] > -1) { - PyList_Append(fragment_idx_list, Py_BuildValue("i", fragments_needed[j])); - j++; - } - break; - } - default: - PyErr_SetString(PyECLibError, "Invalid EC type used in pyeclib.get_required_fragments"); - break; + + while (fragments_needed[i] > -1) { + PyList_Append(fragment_idx_list, Py_BuildValue("i", fragments_needed[i])); + i++; } exit: @@ -1630,156 +469,63 @@ pyeclib_c_reconstruct(PyObject *self, PyObject *args) { PyObject *pyeclib_obj_handle = NULL; pyeclib_t *pyeclib_handle = NULL; - PyObject *data_list = NULL; /* param, list of data fragments */ - PyObject *parity_list = NULL; /* param, list of parity fragments */ - PyObject *missing_idx_list = NULL; /* param, list of missing indexes */ + PyObject *fragments = NULL; /* param, list of fragments */ PyObject *reconstructed = NULL; /* reconstructed object to return */ - int *erased = NULL; /* jerasure notation of erased devs */ - int fragment_size; /* param, size in bytes of fragment */ - int blocksize; /* size in bytes, fragment - header */ - char **data = NULL; /* k length array of data buffers */ - char **parity = NULL; /* m length array of parity buffers */ - int *missing_idxs = NULL; /* array of missing indexes */ + char * c_reconstructed = NULL; /* C string of reconstructed fragment */ + int fragment_len; /* param, size in bytes of fragment */ + int num_fragments; /* number of fragments passed in */ + char **c_fragments = NULL; /* C array containing the fragment payloads */ int destination_idx; /* param, index to reconstruct */ - unsigned long long realloc_bm = 0; /* bitmap, which fragments were realloc'ed */ - int orig_data_size = -1; /* data size (B),from fragment hdr */ - int missing_size; /* number of missing indexes */ - int *decoding_matrix = NULL; /* reconstruct specific decode matrix */ - int *decoding_row = NULL; /* required row from decode matrix */ - int *dm_ids = NULL; /* k length array of surviving indexes */ - int k, m, w; /* EC algorithm parameters */ int ret; /* decode matrix creation return val */ int i = 0; /* a counter */ /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "OOOOii", &pyeclib_obj_handle, &data_list, - &parity_list, &missing_idx_list, &destination_idx, - &fragment_size)) { - PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode"); + if (!PyArg_ParseTuple(args, "OOii", &pyeclib_obj_handle, &fragments, + &fragment_len, &destination_idx)) { + PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.reconstruct"); return NULL; } pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { - PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); - return NULL; - } - k = pyeclib_handle->k; - m = pyeclib_handle->m; - w = pyeclib_handle->w; - if (!PyList_Check(data_list) || !PyList_Check(parity_list) || !PyList_Check(missing_idx_list)) { - PyErr_SetString(PyECLibError, "Invalid structure passed in for data, parity and/or missing_idx list"); + PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.reconstruct"); return NULL; } - if (k != PyList_Size(data_list)) { - PyErr_SetString(PyECLibError, "The data list does not have the correct number of entries"); - return NULL; - } - if (m != PyList_Size(parity_list)) { - PyErr_SetString(PyECLibError, "The parity list does not have the correct number of entries"); + + /* Pre-processing Python data structures */ + if (!PyList_Check(fragments)) { + PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list"); return NULL; } - /* Allocate data structures needed for reconstruction */ - blocksize = FRAGSIZE_2_BLOCKSIZE(fragment_size); - missing_size = (int) PyList_Size(missing_idx_list); - missing_idxs = (int *) alloc_zeroed_buffer(sizeof(int) * (missing_size + 1)); - data = (char **) alloc_zeroed_buffer(sizeof(char *) * k); - parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m); - if (NULL == missing_idxs || NULL == data || NULL == parity) { + num_fragments = PyList_Size(fragments); + + c_fragments = (char **) alloc_zeroed_buffer(sizeof(char *) * num_fragments); + if (NULL == c_fragments) { goto error; } - - /* Prepare for decoding, no need to go further on error */ - if (get_decoding_info(pyeclib_handle, data_list, parity_list, - missing_idx_list, data, parity, missing_idxs, - &orig_data_size, fragment_size, &realloc_bm)) { - PyErr_SetString(PyECLibError, "Could not extract adequate decoding info from data, parity and missing lists"); + + c_reconstructed = (char*) alloc_zeroed_buffer(sizeof(char) * fragment_len); + if (NULL == c_fragments) { goto error; } - /* Create the decoding matrix, and attempt reconstruction */ - erased = jerasure_erasures_to_erased(k, m, missing_idxs); - switch (pyeclib_handle->type) { - case PYECC_RS_CAUCHY_ORIG: - if (destination_idx < k) { - decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int*) * k * k * w * w); - dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k); - if (NULL == decoding_matrix || NULL == dm_ids) { - goto error; - } - ret = jerasure_make_decoding_bitmatrix(k, m, w, pyeclib_handle->bitmatrix, - erased, decoding_matrix, dm_ids); - decoding_row = decoding_matrix + (destination_idx * k * w * w); - } else { - ret = 0; - decoding_row = pyeclib_handle->bitmatrix + ((destination_idx - k) * k * w * w); - } - - if (ret == 0) { - jerasure_bitmatrix_dotprod(k, w, decoding_row, dm_ids, destination_idx, - data, parity, blocksize, - PYECC_CAUCHY_PACKETSIZE); - } - break; - case PYECC_RS_VAND: - if (destination_idx < k) { - decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int*) * k * k); - dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k); - if (NULL == decoding_matrix || NULL == dm_ids) { - goto error; - } - ret = jerasure_make_decoding_matrix(k, m, w, pyeclib_handle->matrix, - erased, decoding_matrix, dm_ids); - decoding_row = decoding_matrix + (destination_idx * k); - } else { - ret = 0; - decoding_row = pyeclib_handle->matrix + ((destination_idx - k) * k); - } - - if (ret == 0) { - jerasure_matrix_dotprod(k, w, decoding_row, dm_ids, destination_idx, - data, parity, blocksize); - } - break; - case PYECC_XOR_HD_3: - case PYECC_XOR_HD_4: - ret = 0; - xor_reconstruct_one(pyeclib_handle->xor_code_desc, data, parity, - missing_idxs, destination_idx, blocksize); - break; - default: - ret = -1; - break; + /* Put the fragments into an array of C strings */ + for (i = 0; i < num_fragments; i++) { + PyObject *tmp_data = PyList_GetItem(fragments, i); + Py_ssize_t len = 0; + PyBytes_AsStringAndSize(tmp_data, &(c_fragments[i]), &len); } - - /* Set the metadata on the reconstructed fragment */ - if (ret == 0) { - char *fragment_ptr = NULL; - if (destination_idx < k) { - fragment_ptr = get_fragment_ptr_from_data_novalidate(data[destination_idx]); - init_fragment_header(fragment_ptr); - set_fragment_idx(fragment_ptr, destination_idx); - set_orig_data_size(fragment_ptr, orig_data_size); - set_fragment_size(fragment_ptr, blocksize); - if (use_inline_chksum(pyeclib_handle)) { - int chksum = crc32(0, data[destination_idx], blocksize); - set_chksum(fragment_ptr, chksum); - } - } else { - fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[destination_idx - k]); - init_fragment_header(fragment_ptr); - set_fragment_idx(fragment_ptr, destination_idx); - set_orig_data_size(fragment_ptr, orig_data_size); - set_fragment_size(fragment_ptr, blocksize); - if (use_inline_chksum(pyeclib_handle)) { - int chksum = crc32(0, parity[destination_idx - k], blocksize); - set_chksum(fragment_ptr, chksum); - } - } - - reconstructed = PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size); - } else { + + ret = liberasurecode_reconstruct_fragment(pyeclib_handle->ec_desc, + c_fragments, + num_fragments, + fragment_len, + destination_idx, + c_reconstructed); + if (ret < 0) { reconstructed = NULL; + } else { + reconstructed = PY_BUILDVALUE_OBJ_LEN(c_reconstructed, fragment_len); } goto out; @@ -1788,34 +534,14 @@ error: reconstructed = NULL; out: - free(erased); - /* Free fragment buffers that needed to be reallocated for alignment */ - for (i = 0; i < k; i++) { - if (realloc_bm & (1 << i)) { - free(get_fragment_ptr_from_data_novalidate(data[i])); - } - } - for (i = 0; i < m; i++) { - if (realloc_bm & (1 << (i + k))) { - free(get_fragment_ptr_from_data_novalidate(parity[i])); - } - } - - check_and_free_buffer(missing_idxs); - check_and_free_buffer(data); - check_and_free_buffer(parity); - check_and_free_buffer(decoding_matrix); - check_and_free_buffer(dm_ids); + check_and_free_buffer(c_fragments); + check_and_free_buffer(c_reconstructed); return reconstructed; } - /** - * Reconstruct all of the missing fragments from a set of fragments. - * - * TODO: There's a lot of duplicated code between this and the - * reconstruct method. Consider refactoring these methods. + * Decode a set of fragments into the original string * * @param pyeclib_obj_handle * @param data_list k length list of data elements @@ -1829,151 +555,68 @@ pyeclib_c_decode(PyObject *self, PyObject *args) { PyObject *pyeclib_obj_handle = NULL; pyeclib_t *pyeclib_handle = NULL; - PyObject *list_of_strips = NULL; /* list of strips to return */ - PyObject *data_list = NULL; /* param, list of data fragments */ - PyObject *parity_list = NULL; /* param, list of data fragments */ - PyObject *missing_idx_list = NULL; /* param, list of missing indexes */ - int fragment_size; /* param, size in bytes of fragment */ - int blocksize; /* size in bytes, fragment - header */ - unsigned long long realloc_bm = 0; /* bitmap, which fragments were realloc'ed */ - char **data = NULL; /* k length array of data buffers */ - char **parity = NULL; /* m length array of parity buffers */ - int *missing_idxs = NULL; /* array of missing indexes */ - int missing_size; /* number of missing indexes */ - int orig_data_size = -1; /* data size in bytes ,from fragment hdr */ - int k, m, w; /* EC algorithm parameters */ - int i = 0, j = 0; /* counters */ + PyObject *fragments = NULL; /* param, list of missing indexes */ + PyObject *orig_payload = NULL; /* buffer to store original payload in */ + int fragment_len; /* param, size in bytes of fragment */ + char **c_fragments = NULL; /* k length array of data buffers */ + int num_fragments; /* param, number of fragments */ + char *c_orig_payload = NULL; /* buffer to store original payload in */ + uint64_t orig_data_size = 0; /* data size in bytes ,from fragment hdr */ + int i = 0; /* counters */ + int ret = 0; /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "OOOOi", &pyeclib_obj_handle, &data_list, &parity_list, &missing_idx_list, &fragment_size)) { - PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode"); + if (!PyArg_ParseTuple(args, "OOi", &pyeclib_obj_handle, &fragments, &fragment_len)) { + PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.decode"); return NULL; } pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { - PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); - return NULL; - } - k = pyeclib_handle->k; - m = pyeclib_handle->m; - w = pyeclib_handle->w; - if (!PyList_Check(data_list) || !PyList_Check(parity_list) || !PyList_Check(missing_idx_list)) { - PyErr_SetString(PyECLibError, "Invalid structure passed in for data, parity and/or missing_idx list"); - return NULL; - } - if (k != PyList_Size(data_list)) { - PyErr_SetString(PyECLibError, "The data list does not have the correct number of entries"); + PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.decode"); return NULL; } - if (m != PyList_Size(parity_list)) { - PyErr_SetString(PyECLibError, "The parity list does not have the correct number of entries"); + if (!PyList_Check(fragments)) { + PyErr_SetString(PyECLibError, "Invalid structure passed in for available fragments in pyeclib.decode"); return NULL; } + + num_fragments = PyList_Size(fragments); - /* Allocate data structures needed for reconstruction */ - blocksize = FRAGSIZE_2_BLOCKSIZE(fragment_size); - missing_size = (int) PyList_Size(missing_idx_list); - missing_idxs = (int *) alloc_zeroed_buffer(sizeof(int) * (missing_size + 1)); - data = (char **) alloc_zeroed_buffer(sizeof(char *) * k); - parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m); - if (NULL == missing_idxs || NULL == data || NULL == parity) { - goto error; - } - - /* Prepare for decoding, no need to go further on error */ - if (get_decoding_info(pyeclib_handle, data_list, parity_list, missing_idx_list, - data, parity, missing_idxs, &orig_data_size, - fragment_size, &realloc_bm)) { - PyErr_SetString(PyECLibError, "Could not extract adequate decoding info from data, parity and missing lists"); + if (pyeclib_handle->ec_args.k > num_fragments) { + PyErr_SetString(PyECLibError, "The fragment list does not have enough entries in pyeclib.decode"); return NULL; } - - /* Reconstruct the missing fragments */ - switch (pyeclib_handle->type) { - case PYECC_RS_CAUCHY_ORIG: - jerasure_bitmatrix_decode(k, m, w, pyeclib_handle->bitmatrix, 0, missing_idxs, - data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE); - break; - case PYECC_XOR_HD_3: - case PYECC_XOR_HD_4: - pyeclib_handle->xor_code_desc->decode(pyeclib_handle->xor_code_desc, data, - parity, missing_idxs, blocksize, 1); - break; - case PYECC_RS_VAND: - default: - jerasure_matrix_decode(k, m, w, pyeclib_handle->matrix, 1, missing_idxs, - data, parity, blocksize); - break; - } - - /* Create the python list to return */ - list_of_strips = PyList_New(k + m); - if (NULL == list_of_strips) { + + c_fragments = (char **) alloc_zeroed_buffer(sizeof(char *) * num_fragments); + if (NULL == c_fragments) { goto error; } - /* Create headers for the newly decoded elements */ - while (missing_idxs[j] >= 0) { - int missing_idx = missing_idxs[j]; - if (missing_idx < k) { - char *fragment_ptr = get_fragment_ptr_from_data_novalidate(data[missing_idx]); - init_fragment_header(fragment_ptr); - set_fragment_idx(fragment_ptr, missing_idx); - set_orig_data_size(fragment_ptr, orig_data_size); - set_fragment_size(fragment_ptr, blocksize); - if (use_inline_chksum(pyeclib_handle)) { - int chksum = crc32(0, data[missing_idx], blocksize); - set_chksum(fragment_ptr, chksum); - } - } else if (missing_idx >= k) { - int parity_idx = missing_idx - k; - char *fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[parity_idx]); - init_fragment_header(fragment_ptr); - set_fragment_idx(fragment_ptr, missing_idx); - set_orig_data_size(fragment_ptr, orig_data_size); - set_fragment_size(fragment_ptr, blocksize); - if (use_inline_chksum(pyeclib_handle)) { - int chksum = crc32(0, parity[parity_idx], blocksize); - set_chksum(fragment_ptr, chksum); - } - } - j++; - } - - /* Fill in the data fragments */ - for (i = 0; i < k; i++) { - char *fragment_ptr = get_fragment_ptr_from_data(data[i]); - PyList_SET_ITEM(list_of_strips, i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); - } - /* Fill in the parity fragments */ - for (i = 0; i < m; i++) { - char *fragment_ptr = get_fragment_ptr_from_data(parity[i]); - PyList_SET_ITEM(list_of_strips, k + i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); + /* Put the fragments into an array of C strings */ + for (i = 0; i < num_fragments; i++) { + PyObject *tmp_data = PyList_GetItem(fragments, i); + Py_ssize_t len = 0; + PyBytes_AsStringAndSize(tmp_data, &(c_fragments[i]), &len); } + ret = liberasurecode_decode(pyeclib_handle->ec_desc, + c_fragments, + num_fragments, + fragment_len, + &c_orig_payload, + &orig_data_size); + + orig_payload = PY_BUILDVALUE_OBJ_LEN(c_orig_payload, orig_data_size); goto exit; error: - list_of_strips = NULL; + orig_payload = NULL; exit: - /* Free fragment buffers that needed to be reallocated for alignment */ - for (i = 0; i < k; i++) { - if (realloc_bm & (1 << i)) { - free(get_fragment_ptr_from_data_novalidate(data[i])); - } - } - for (i = 0; i < m; i++) { - if (realloc_bm & (1 << (i + k))) { - free(get_fragment_ptr_from_data_novalidate(parity[i])); - } - } - - check_and_free_buffer(missing_idxs); - check_and_free_buffer(data); - check_and_free_buffer(parity); + check_and_free_buffer(c_fragments); + check_and_free_buffer(c_orig_payload); - return list_of_strips; + return orig_payload; } @@ -1990,14 +633,14 @@ pyeclib_c_get_metadata(PyObject *self, PyObject *args) { PyObject *pyeclib_obj_handle = NULL; pyeclib_t* pyeclib_handle = NULL; - char *data = NULL; /* param, fragment from caller */ - int data_len; /* param, data len (B) */ - int metadata_len; /* metadata header size (B) */ - fragment_metadata_t *fragment_metadata = NULL; /* buffer to hold metadata */ - PyObject *ret_fragment_metadata = NULL; /* metadata object to return */ + char *fragment = NULL; /* param, fragment from caller */ + int fragment_metadata_len; /* metadata header size (B) */ + char *c_fragment_metadata = NULL; /* buffer to hold metadata */ + PyObject *fragment_metadata = NULL; /* metadata object to return */ + int ret; /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &data, &data_len)) { + if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &fragment)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_metadata"); return NULL; } @@ -2007,19 +650,17 @@ pyeclib_c_get_metadata(PyObject *self, PyObject *args) return NULL; } - /* Obtain the metadata from the data and build a object to return */ - metadata_len = sizeof(fragment_metadata_t); - fragment_metadata = (fragment_metadata_t *) alloc_zeroed_buffer(metadata_len); - if (NULL == fragment_metadata) { - ret_fragment_metadata = NULL; + ret = liberasurecode_get_fragment_metadata(fragment, &c_fragment_metadata, &fragment_metadata_len); + + if (ret < 0) { + fragment_metadata = NULL; + PyErr_SetString(PyECLibError, "Failed to get metadata in pyeclib.get_metadata"); } else { - get_fragment_metadata(pyeclib_handle, data, fragment_metadata); - ret_fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)fragment_metadata, - metadata_len); - check_and_free_buffer(fragment_metadata); + fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)c_fragment_metadata, + fragment_metadata_len); } - return ret_fragment_metadata; + return fragment_metadata; } @@ -2039,17 +680,17 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args) PyObject *pyeclib_obj_handle = NULL; pyeclib_t *pyeclib_handle = NULL; PyObject *fragment_metadata_list = NULL; /* param, fragment metadata */ - fragment_metadata_t **c_fragment_metadata_list = NULL; /* c version of metadata */ + char **c_fragment_metadata_list = NULL; /* c version of metadata */ + int fragment_metadata_size; /* size of the metadata payload */ int num_fragments; /* k + m from EC algorithm */ - char **c_fragment_signatures = NULL; /* array of alg. signatures */ - int k, m, w; /* EC algorithm params */ + int k, m; /* EC algorithm params */ int size; /* size for buf allocation */ int ret = -1; /* c return value */ PyObject *ret_obj = NULL; /* python long to return */ int i = 0; /* a counter */ /* Obtain and validate the method parameters */ - if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_metadata_list)) { + if (!PyArg_ParseTuple(args, "OOi", &pyeclib_obj_handle, &fragment_metadata_list, &fragment_metadata_size)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode"); return NULL; } @@ -2058,9 +699,8 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args) PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); return NULL; } - k = pyeclib_handle->k; - m = pyeclib_handle->m; - w = pyeclib_handle->w; + k = pyeclib_handle->ec_args.k; + m = pyeclib_handle->ec_args.m; num_fragments = k + m; if (num_fragments != PyList_Size(fragment_metadata_list)) { PyErr_SetString(PyECLibError, "Not enough fragment metadata to perform integrity check"); @@ -2068,95 +708,29 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args) } /* Allocate space for fragment signatures */ - size = sizeof(fragment_metadata_t * ) * num_fragments; - c_fragment_metadata_list = (fragment_metadata_t**) alloc_zeroed_buffer(size); - size = sizeof(char *) * num_fragments; - c_fragment_signatures = (char **) alloc_zeroed_buffer(size); - if (NULL == c_fragment_metadata_list || NULL == c_fragment_signatures) { + size = sizeof(char * ) * num_fragments; + c_fragment_metadata_list = (char **) alloc_zeroed_buffer(size); + if (NULL == c_fragment_metadata_list) { goto error; } - /* Populate and order the metadata */ + /* Populate the metadata into a C array */ for (i = 0; i < num_fragments; i++) { PyObject *tmp_data = PyList_GetItem(fragment_metadata_list, i); Py_ssize_t len = 0; - char *c_buf = NULL; - fragment_metadata_t *fragment_metadata; - PyBytes_AsStringAndSize(tmp_data, &(c_buf), &len); - - fragment_metadata = (fragment_metadata_t*)c_buf; - c_fragment_metadata_list[fragment_metadata->idx] = fragment_metadata; - - if (supports_alg_sig(pyeclib_handle)) { - c_fragment_signatures[fragment_metadata->idx] = (char*)alloc_aligned_buffer16(PYCC_MAX_SIG_LEN); - memcpy(c_fragment_signatures[fragment_metadata->idx], fragment_metadata->signature, PYCC_MAX_SIG_LEN); - } else { - c_fragment_signatures[fragment_metadata->idx] = fragment_metadata->signature; - } + PyBytes_AsStringAndSize(tmp_data, &(c_fragment_metadata_list[i]), &len); } - - /* Ensure all fragments are here and check integrity using alg signatures */ - if (supports_alg_sig(pyeclib_handle)) { - char **parity_sigs = (char **) alloc_zeroed_buffer(sizeof(char **) * m); - if (NULL == parity_sigs) { - goto error; - } - for (i = 0; i < m; i++) { - parity_sigs[i] = (char *) alloc_aligned_buffer16(PYCC_MAX_SIG_LEN); - if (NULL == parity_sigs[i]) { - int j; - for (j = 0; j < i; j++) free(parity_sigs[j]); - goto error; - } else { - memset(parity_sigs[i], 0, PYCC_MAX_SIG_LEN); - } - } - - /* Calculate the parity of the signatures */ - if (pyeclib_handle->type == PYECC_RS_VAND) { - jerasure_matrix_encode(k, m, w, pyeclib_handle->matrix, - c_fragment_signatures, parity_sigs, PYCC_MAX_SIG_LEN); - } else { - pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc, - c_fragment_signatures, - parity_sigs, - PYCC_MAX_SIG_LEN); - } - - /* Compare the parity of the signatures, and the signature of the parity */ - for (i = 0; i < m; i++) { - if (memcmp(parity_sigs[i], c_fragment_signatures[k + i], PYCC_MAX_SIG_LEN) != 0) { - ret = i; - break; - } - } - - /* Clean up memory used in algebraic signature checking */ - for (i = 0; i < m; i++) { - free(parity_sigs[i]); - } - check_and_free_buffer(parity_sigs); - for (i = 0; i < k; i++) { - free(c_fragment_signatures[i]); - } - } else if (use_inline_chksum(pyeclib_handle)) { - for (i = 0; i < num_fragments; i++) { - if (c_fragment_metadata_list[i]->chksum_mismatch == 1) { - ret = i; - break; - } - } - } - - /* Return index of first checksum signature error */ + + ret = liberasurecode_verify_stripe_metadata(c_fragment_metadata_list, num_fragments, fragment_metadata_size); + ret_obj = PyLong_FromLong((long)ret); + goto exit; error: ret_obj = NULL; exit: - free(c_fragment_signatures); free(c_fragment_metadata_list); return ret_obj; @@ -2168,8 +742,6 @@ static PyMethodDef PyECLibMethods[] = { {"encode", pyeclib_c_encode, METH_VARARGS, "Create parity using source data"}, {"decode", pyeclib_c_decode, METH_VARARGS, "Recover all lost data/parity"}, {"reconstruct", pyeclib_c_reconstruct, METH_VARARGS, "Recover selective data/parity"}, - {"fragments_to_string", pyeclib_c_fragments_to_string, METH_VARARGS, "Try to transform a set of fragments into original string without calling the decoder"}, - {"get_fragment_partition", pyeclib_c_get_fragment_partition, METH_VARARGS, "Parition fragments into data and parity, also returns a list of missing indexes"}, {"get_required_fragments", pyeclib_c_get_required_fragments, METH_VARARGS, "Return the fragments required to reconstruct a set of missing fragments"}, {"get_segment_info", pyeclib_c_get_segment_info, METH_VARARGS, "Return segment and fragment size information needed when encoding a segmented stream"}, {"get_metadata", pyeclib_c_get_metadata, METH_VARARGS, "Get the integrity checking metadata for a fragment"}, diff --git a/src/c/pyeclib_c/pyeclib_c.h b/src/c/pyeclib_c/pyeclib_c.h index f7d8439..4b7b3db 100644 --- a/src/c/pyeclib_c/pyeclib_c.h +++ b/src/c/pyeclib_c/pyeclib_c.h @@ -30,122 +30,14 @@ /* For exact-width integer types */ #include <stdint.h> -/* - * Make sure these enum values match those exposed from the Python EC interface - * src/python/pyeclib/ec_iface.py - */ -#define PYECLIB_MAX_DATA 32 -#define PYECLIB_MAX_PARITY 32 - -typedef enum { - PYECC_NOT_FOUND = 0, - PYECC_RS_VAND = 1, - PYECC_RS_CAUCHY_ORIG = 2, - PYECC_XOR_HD_3 = 3, - PYECC_XOR_HD_4 = 4, - PYECC_NUM_TYPES = 4, -} pyeclib_type_t; - -const char *pyeclib_type_str[] = { - "not_found", - "jerasure_rs_vand", - "jerasure_rs_cauchy_orig", - "flat_xor_3", - "flat_xor_4", -}; - -const int pyeclib_type_word_size_bytes[] = { - 0, - sizeof(long), /* rs_vand */ - sizeof(long), /* rs_cauchy_orig */ - sizeof(long), /* flat_xor_3 */ - sizeof(long) /* flat_xor_4 */ -}; - -const int pyeclib_type_best_w_size_bytes[] = { - 32, /* default */ - 16, /* rs_vand */ - 4, /* rs_cauchy_orig */ - 32, /* flat_xor_3 */ - 32 /* flat_xor_4 */ -}; - -// Unconditionally enforce alignment for now. -// This is needed for the SIMD extentions. -// TODO (kmg): Parse cpuinfo and determine if it is necessary... -const int pyeclib_type_needs_addr_align[] = { - -1, - 1, - 1, - 1, - 1 -}; - typedef struct pyeclib_s { - int k; - int m; - int w; - int *matrix; - int *bitmatrix; - int **schedule; - xor_code_t *xor_code_desc; - alg_sig_t *alg_sig_desc; - pyeclib_type_t type; - int inline_chksum; - int algsig_chksum; + int ec_desc; + struct ec_args ec_args; } pyeclib_t; -/* - * Convert the string ECC type to the enum value - * Start lookup at index 1 - */ -static inline -pyeclib_type_t get_ecc_type(const char *str_type) -{ - int i; - for (i = 1; i <= PYECC_NUM_TYPES; i++) { - if (strcmp(str_type, pyeclib_type_str[i]) == 0) { - return i; - } - } - return PYECC_NOT_FOUND; -} -/* - * Convert the string ECC type to the w value - * Start lookup at index 1 - */ -static inline -unsigned int get_best_w_for_ecc_type(pyeclib_type_t type) -{ - if (type > PYECC_NUM_TYPES) - return -1; - - return pyeclib_type_best_w_size_bytes[type]; -} - - -#define PYECC_FLAGS_MASK 0x1 -#define PYECC_FLAGS_READ_VERIFY 0x1 #define PYECC_HANDLE_NAME "pyeclib_handle" -#define PYECC_CAUCHY_PACKETSIZE (sizeof(long) * 128) - -#define PYCC_MAX_SIG_LEN 8 - -typedef struct fragment_metadata_s -{ - int size; - int idx; - char signature[PYCC_MAX_SIG_LEN]; - int chksum_mismatch; -} fragment_metadata_t; - -#define FRAGSIZE_2_BLOCKSIZE(fragment_size) (fragment_size - sizeof(fragment_header_t)) - -#define PYECLIB_WORD_SIZE(type) pyeclib_type_word_size_bytes[type] - -#define PYECLIB_NEEDS_ADDR_ALIGNMENT(type) pyeclib_type_needs_addr_align[type] #endif diff --git a/src/python/pyeclib/core.py b/src/python/pyeclib/core.py index dd1c4db..4385d0a 100644 --- a/src/python/pyeclib/core.py +++ b/src/python/pyeclib/core.py @@ -47,6 +47,7 @@ class ECPyECLibDriver(object): self.m = m self.ec_type = ec_type self.chksum_type = chksum_type + hd = m self.inline_chksum = 0 self.algsig_chksum = 0 @@ -56,39 +57,53 @@ class ECPyECLibDriver(object): elif self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.algsig: self.algsig_chksum = 1 + name = self.ec_type.name + + if name == "flat_xor_hd_3": + hd = 3 + name = "flat_xor_hd" + elif name == "flat_xor_hd_4": + hd = 4 + name = "flat_xor_hd" + self.handle = pyeclib_c.init( self.k, self.m, - self.ec_type.name, + name, + hd, self.inline_chksum, self.algsig_chksum) def encode(self, data_bytes): return pyeclib_c.encode(self.handle, data_bytes) + def _validate_and_return_fragment_size(self, fragments): + if len(fragments) > 0 and len(fragments[0]) == 0: + return -1 + fragment_len = len(fragments[0]) + + for fragment in fragments[1:]: + if len(fragment) != fragment_len: + return -1 + + return fragment_len + def decode(self, fragment_payloads): - try: - ret_string = pyeclib_c.fragments_to_string( - self.handle, - fragment_payloads) - except Exception as e: - raise ECPyECLibException("Error in ECPyECLibDriver.decode") - - if ret_string is None: - (data_frags, - parity_frags, - missing_idxs) = pyeclib_c.get_fragment_partition( - self.handle, fragment_payloads) - decoded_fragments = pyeclib_c.decode( - self.handle, data_frags, parity_frags, missing_idxs, - len(data_frags[0])) - ret_string = pyeclib_c.fragments_to_string( - self.handle, - decoded_fragments) + fragment_len = self._validate_and_return_fragment_size(fragment_payloads) + if fragment_len < 0: + raise ECPyECLibException("Invalid fragment payload in ECPyECLibDriver.decode") - return ret_string + if len(fragment_payloads) < self.k: + raise ECPyECLibException("Not enough fragments given in ECPyECLibDriver.decode") + + return pyeclib_c.decode(self.handle, fragment_payloads, fragment_len) + def reconstruct(self, fragment_payloads, indexes_to_reconstruct): + fragment_len = self._validate_and_return_fragment_size(fragment_payloads) + if fragment_len < 0: + raise ECPyECLibException("Invalid fragment payload in ECPyECLibDriver.reconstruct") + reconstructed_data = [] # Reconstruct the data, then the parity @@ -99,13 +114,8 @@ class ECPyECLibDriver(object): while len(_indexes_to_reconstruct) > 0: index = _indexes_to_reconstruct.pop(0) - (data_frags, - parity_frags, - missing_idxs) = pyeclib_c.get_fragment_partition( - self.handle, fragment_payloads) reconstructed = pyeclib_c.reconstruct( - self.handle, data_frags, parity_frags, missing_idxs, - index, len(data_frags[0])) + self.handle, fragment_payloads, index, fragment_len) reconstructed_data.append(reconstructed) return reconstructed_data @@ -118,7 +128,11 @@ class ECPyECLibDriver(object): return pyeclib_c.get_metadata(self.handle, fragment) def verify_stripe_metadata(self, fragment_metadata_list): - return pyeclib_c.check_metadata(self.handle, fragment_metadata_list) + metadata_len = self._validate_and_return_fragment_size(fragment_payloads) + if metadata_len < 0: + raise ECPyECLibException("Invalid fragment payload in ECPyECLibDriver.verify_metadata") + + return pyeclib_c.check_metadata(self.handle, fragment_metadata_list, metadata_len) def get_segment_info(self, data_len, segment_size): return pyeclib_c.get_segment_info(self.handle, data_len, segment_size) diff --git a/src/python/pyeclib/ec_iface.py b/src/python/pyeclib/ec_iface.py index a80d895..5dde2b4 100644 --- a/src/python/pyeclib/ec_iface.py +++ b/src/python/pyeclib/ec_iface.py @@ -82,10 +82,10 @@ class PyECLibEnum(Enum): class PyECLib_EC_Types(PyECLibEnum): # Note: the Enum start value defaults to 1 as the starting value and not 0 # 0 is False in the boolean sense but enum members evaluate to True - rs_vand = 1 - rs_cauchy_orig = 2 - flat_xor_3 = 3 - flat_xor_4 = 4 + jerasure_rs_vand = 1 + jerasure_rs_cauchy = 2 + flat_xor_hd_3 = 3 + flat_xor_hd_4 = 4 # Output of Erasure (en)Coding process are data "fragments". Fragment data |