summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKevin Greenan <kmgreen2@gmail.com>2015-05-19 09:09:46 -0700
committerKevin Greenan <kmgreen2@gmail.com>2015-06-18 12:31:17 -0700
commite9ca485c7c62aa952d0a3c564e0fdcf530740d9d (patch)
tree74d56c4c3e0c1a278f5e34f3367d470f0150db12
parentfe537605b318f88bbe36459d4772bb9d8afa052f (diff)
downloadliberasurecode-e9ca485c7c62aa952d0a3c564e0fdcf530740d9d.tar.gz
Plugging new internal RS backend into liberasurecode.
-rw-r--r--include/erasurecode/erasurecode.h13
-rw-r--r--src/Makefile.am1
-rw-r--r--src/backends/rs_vand/rs_vand.c312
-rw-r--r--src/erasurecode.c2
-rw-r--r--test/liberasurecode_test.c109
5 files changed, 430 insertions, 7 deletions
diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h
index 4604c2a..e40c164 100644
--- a/include/erasurecode/erasurecode.h
+++ b/include/erasurecode/erasurecode.h
@@ -40,12 +40,13 @@ extern "C" {
/* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */
typedef enum {
- EC_BACKEND_NULL = 0,
- EC_BACKEND_JERASURE_RS_VAND = 1,
- EC_BACKEND_JERASURE_RS_CAUCHY = 2,
- EC_BACKEND_FLAT_XOR_HD = 3,
- EC_BACKEND_ISA_L_RS_VAND = 4,
- EC_BACKEND_SHSS = 5,
+ EC_BACKEND_NULL = 0,
+ EC_BACKEND_JERASURE_RS_VAND = 1,
+ EC_BACKEND_JERASURE_RS_CAUCHY = 2,
+ EC_BACKEND_FLAT_XOR_HD = 3,
+ EC_BACKEND_ISA_L_RS_VAND = 4,
+ EC_BACKEND_SHSS = 5,
+ EC_BACKEND_INTERNAL_RS_VAND = 6,
EC_BACKENDS_MAX,
} ec_backend_id_t;
diff --git a/src/Makefile.am b/src/Makefile.am
index 1a837c6..63b2542 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -21,6 +21,7 @@ liberasurecode_la_SOURCES = \
backends/jerasure/jerasure_rs_vand.c \
backends/jerasure/jerasure_rs_cauchy.c \
backends/isa-l/isa_l_rs_vand.c \
+ backends/rs_vand/rs_vand.c \
backends/shss/shss.c
# Install additional header files
diff --git a/src/backends/rs_vand/rs_vand.c b/src/backends/rs_vand/rs_vand.c
new file mode 100644
index 0000000..b709e13
--- /dev/null
+++ b/src/backends/rs_vand/rs_vand.c
@@ -0,0 +1,312 @@
+/*
+ * Copyright 2015 Kevin M Greenan
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this
+ * list of conditions and the following disclaimer in the documentation and/or
+ * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
+ * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * vi: set noai tw=79 ts=4 sw=4:
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "erasurecode.h"
+#include "erasurecode_backend.h"
+#include "erasurecode_helpers.h"
+
+#define INTERNAL_RS_VAND_LIB_MAJOR 1
+#define INTERNAL_RS_VAND_LIB_MINOR 0
+#define INTERNAL_RS_VAND_LIB_REV 0
+#define INTERNAL_RS_VAND_LIB_VER_STR "1.0"
+#define INTERNAL_RS_VAND_LIB_NAME "liberasurecode_rsvand"
+#if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__)
+#define INTERNAL_RS_VAND_SO_NAME "liberasurecode_rsvand.dylib"
+#else
+#define INTERNAL_RS_VAND_SO_NAME "liberasurecode_rsvand.so"
+#endif
+
+/* Forward declarations */
+struct ec_backend_op_stubs internal_rs_vand_ops;
+struct ec_backend internal_rs_vand;
+struct ec_backend_common backend_internal_rs_vand;
+
+typedef int (*internal_rs_vand_encode_func)(int *, char **, char **, int, int, int);
+typedef int (*internal_rs_vand_decode_func)(int *, char **, char **, int, int, int *, int, int);
+typedef int (*internal_rs_vand_reconstruct_func)(int *, char **, char **, int, int, int *, int, int);
+typedef void (*init_rs_vand_func)(int, int);
+typedef void (*deinit_rs_vand_func)();
+typedef void (*free_systematic_matrix_func)(int *);
+typedef int* (*make_systematic_matrix_func)(int, int);
+
+
+struct internal_rs_vand_descriptor {
+ /* calls required for init */
+ init_rs_vand_func init_rs_vand;
+ deinit_rs_vand_func deinit_rs_vand;
+ free_systematic_matrix_func free_systematic_matrix;
+ make_systematic_matrix_func make_systematic_matrix;
+
+ /* calls required for encode */
+ internal_rs_vand_encode_func internal_rs_vand_encode;
+
+ /* calls required for decode */
+ internal_rs_vand_decode_func internal_rs_vand_decode;
+
+ /* calls required for reconstruct */
+ internal_rs_vand_reconstruct_func internal_rs_vand_reconstruct;
+
+ /* fields needed to hold state */
+ int *matrix;
+ int k;
+ int m;
+ int w;
+};
+
+static int internal_rs_vand_encode(void *desc, char **data, char **parity,
+ int blocksize)
+{
+ struct internal_rs_vand_descriptor *rs_vand_desc =
+ (struct internal_rs_vand_descriptor*) desc;
+
+ /* FIXME: Should this return something? */
+ rs_vand_desc->internal_rs_vand_encode(rs_vand_desc->matrix, data, parity,
+ rs_vand_desc->k, rs_vand_desc->m, blocksize);
+ return 0;
+}
+
+static int internal_rs_vand_decode(void *desc, char **data, char **parity,
+ int *missing_idxs, int blocksize)
+{
+ struct internal_rs_vand_descriptor *rs_vand_desc =
+ (struct internal_rs_vand_descriptor*) desc;
+
+ /* FIXME: Should this return something? */
+ rs_vand_desc->internal_rs_vand_decode(rs_vand_desc->matrix, data, parity,
+ rs_vand_desc->k, rs_vand_desc->m, missing_idxs, blocksize, 1);
+
+ return 0;
+}
+
+static int internal_rs_vand_reconstruct(void *desc, char **data, char **parity,
+ int *missing_idxs, int destination_idx, int blocksize)
+{
+ struct internal_rs_vand_descriptor *rs_vand_desc =
+ (struct internal_rs_vand_descriptor*) desc;
+
+ /* FIXME: Should this return something? */
+ rs_vand_desc->internal_rs_vand_reconstruct(rs_vand_desc->matrix, data, parity,
+ rs_vand_desc->k, rs_vand_desc->m, missing_idxs, destination_idx, blocksize);
+
+ return 0;
+}
+
+static int internal_rs_vand_min_fragments(void *desc, int *missing_idxs,
+ int *fragments_to_exclude, int *fragments_needed)
+{
+ struct internal_rs_vand_descriptor *rs_vand_desc =
+ (struct internal_rs_vand_descriptor*)desc;
+
+ uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
+ uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
+ int i;
+ int j = 0;
+ int ret = -1;
+
+ for (i = 0; i < (rs_vand_desc->k + rs_vand_desc->m); i++) {
+ if (!(missing_bm & (1 << i))) {
+ fragments_needed[j] = i;
+ j++;
+ }
+ if (j == rs_vand_desc->k) {
+ ret = 0;
+ fragments_needed[j] = -1;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+static void * internal_rs_vand_init(struct ec_backend_args *args,
+ void *backend_sohandle)
+{
+ struct internal_rs_vand_descriptor *desc = NULL;
+
+ desc = (struct internal_rs_vand_descriptor *)
+ malloc(sizeof(struct internal_rs_vand_descriptor));
+ if (NULL == desc) {
+ return NULL;
+ }
+
+ desc->k = args->uargs.k;
+ desc->m = args->uargs.m;
+
+ /* store w back in args so upper layer can get to it */
+ desc->w = 16; // w is currently hard-coded at 16
+
+ // This check should not matter, since 64K is way higher
+ // than anyone should ever use
+ if ((desc->k + desc->m) > 65536) {
+ goto error;
+ }
+
+ /*
+ * ISO C forbids casting a void* to a function pointer.
+ * Since dlsym return returns a void*, we use this union to
+ * "transform" the void* to a function pointer.
+ */
+ union {
+ init_rs_vand_func initp;
+ deinit_rs_vand_func deinitp;
+ free_systematic_matrix_func freematrixp;
+ make_systematic_matrix_func makematrixp;
+ internal_rs_vand_encode_func encodep;
+ internal_rs_vand_decode_func decodep;
+ internal_rs_vand_reconstruct_func reconstructp;
+ void *vptr;
+ } func_handle = {.vptr = NULL};
+
+
+ /* fill in function addresses */
+ func_handle.vptr = NULL;
+ func_handle.vptr = dlsym(backend_sohandle, "init_rs_vand");
+ desc->init_rs_vand = func_handle.initp;
+ if (NULL == desc->init_rs_vand) {
+ goto error;
+ }
+
+ func_handle.vptr = NULL;
+ func_handle.vptr = dlsym(backend_sohandle, "deinit_rs_vand");
+ desc->deinit_rs_vand = func_handle.deinitp;
+ if (NULL == desc->deinit_rs_vand) {
+ goto error;
+ }
+
+ func_handle.vptr = NULL;
+ func_handle.vptr = dlsym(backend_sohandle, "make_systematic_matrix");
+ desc->make_systematic_matrix = func_handle.makematrixp;
+ if (NULL == desc->make_systematic_matrix) {
+ goto error;
+ }
+
+ func_handle.vptr = NULL;
+ func_handle.vptr = dlsym(backend_sohandle, "free_systematic_matrix");
+ desc->free_systematic_matrix = func_handle.freematrixp;
+ if (NULL == desc->free_systematic_matrix) {
+ goto error;
+ }
+
+ func_handle.vptr = NULL;
+ func_handle.vptr = dlsym(backend_sohandle, "internal_rs_vand_encode");
+ desc->internal_rs_vand_encode = func_handle.encodep;
+ if (NULL == desc->internal_rs_vand_encode) {
+ goto error;
+ }
+
+ func_handle.vptr = NULL;
+ func_handle.vptr = dlsym(backend_sohandle, "internal_rs_vand_decode");
+ desc->internal_rs_vand_decode = func_handle.decodep;
+ if (NULL == desc->internal_rs_vand_decode) {
+ goto error;
+ }
+
+ func_handle.vptr = NULL;
+ func_handle.vptr = dlsym(backend_sohandle, "internal_rs_vand_reconstruct");
+ desc->internal_rs_vand_reconstruct = func_handle.reconstructp;
+ if (NULL == desc->internal_rs_vand_reconstruct) {
+ goto error;
+ }
+
+ desc->init_rs_vand(desc->k, desc->m);
+
+ desc->matrix = desc->make_systematic_matrix(desc->k, desc->m);
+
+ if (NULL == desc->matrix) {
+ goto error;
+ }
+
+ return desc;
+
+error:
+ free(desc);
+
+ return NULL;
+}
+
+/**
+ * Return the element-size, which is the number of bits stored
+ * on a given device, per codeword. For Vandermonde, this is
+ * 'w'. For somthing like cauchy, this is packetsize * w.
+ *
+ * Returns the size in bits!
+ */
+static int
+internal_rs_vand_element_size(void* desc)
+{
+ struct internal_rs_vand_descriptor *rs_vand_desc = NULL;
+
+ rs_vand_desc = (struct internal_rs_vand_descriptor*) desc;
+
+ return rs_vand_desc->w;
+}
+
+static int internal_rs_vand_exit(void *desc)
+{
+ struct internal_rs_vand_descriptor *rs_vand_desc = NULL;
+
+ rs_vand_desc = (struct internal_rs_vand_descriptor*) desc;
+
+ rs_vand_desc->free_systematic_matrix(rs_vand_desc->matrix);
+ rs_vand_desc->deinit_rs_vand();
+ free(rs_vand_desc);
+
+ return 0;
+}
+
+/*
+ * For the time being, we only claim compatibility with versions that
+ * match exactly
+ */
+static bool internal_rs_vand_is_compatible_with(uint32_t version) {
+ return version == backend_internal_rs_vand.ec_backend_version;
+}
+
+struct ec_backend_op_stubs internal_rs_vand_op_stubs = {
+ .INIT = internal_rs_vand_init,
+ .EXIT = internal_rs_vand_exit,
+ .ENCODE = internal_rs_vand_encode,
+ .DECODE = internal_rs_vand_decode,
+ .FRAGSNEEDED = internal_rs_vand_min_fragments,
+ .RECONSTRUCT = internal_rs_vand_reconstruct,
+ .ELEMENTSIZE = internal_rs_vand_element_size,
+ .ISCOMPATIBLEWITH = internal_rs_vand_is_compatible_with,
+};
+
+struct ec_backend_common backend_internal_rs_vand = {
+ .id = EC_BACKEND_INTERNAL_RS_VAND,
+ .name = INTERNAL_RS_VAND_LIB_NAME,
+ .soname = INTERNAL_RS_VAND_SO_NAME,
+ .soversion = INTERNAL_RS_VAND_LIB_VER_STR,
+ .ops = &internal_rs_vand_op_stubs,
+ .backend_metadata_size = 0,
+ .ec_backend_version = _VERSION(INTERNAL_RS_VAND_LIB_MAJOR,
+ INTERNAL_RS_VAND_LIB_MINOR,
+ INTERNAL_RS_VAND_LIB_REV),
+};
diff --git a/src/erasurecode.c b/src/erasurecode.c
index d9a868c..3c9e123 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -44,6 +44,7 @@ extern struct ec_backend_common backend_jerasure_rs_vand;
extern struct ec_backend_common backend_jerasure_rs_cauchy;
extern struct ec_backend_common backend_isa_l_rs_vand;
extern struct ec_backend_common backend_shss;
+extern struct ec_backend_common backend_internal_rs_vand;
ec_backend_t ec_backends_supported[] = {
(ec_backend_t) &backend_null,
@@ -52,6 +53,7 @@ ec_backend_t ec_backends_supported[] = {
(ec_backend_t) &backend_flat_xor_hd,
(ec_backend_t) &backend_isa_l_rs_vand,
(ec_backend_t) &backend_shss,
+ (ec_backend_t) &backend_internal_rs_vand,
NULL,
};
diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c
index c9d67af..9f1ad9d 100644
--- a/test/liberasurecode_test.c
+++ b/test/liberasurecode_test.c
@@ -39,6 +39,7 @@
#define JERASURE_RS_CAUCHY_BACKEND "jerasure_rs_cauchy"
#define ISA_L_RS_VAND_BACKEND "isa_l_rs_vand"
#define SHSS_BACKEND "shss"
+#define RS_VAND_BACKEND "rs_vand"
typedef void (*TEST_FUNC)();
@@ -180,12 +181,52 @@ struct ec_args shss_args = {
struct ec_args *shss_test_args[] = { &shss_args, NULL };
+struct ec_args internal_rs_vand_args = {
+ .k = 10,
+ .m = 4,
+ .w = 16,
+ .hd = 5,
+ .ct = CHKSUM_NONE,
+};
+
+struct ec_args internal_rs_vand_44_args = {
+ .k = 4,
+ .m = 4,
+ .w = 16,
+ .hd = 5,
+ .ct = CHKSUM_NONE,
+};
+
+struct ec_args internal_rs_vand_48_args = {
+ .k = 4,
+ .m = 8,
+ .w = 16,
+ .hd = 9,
+ .ct = CHKSUM_NONE,
+};
+
+struct ec_args internal_rs_vand_1010_args = {
+ .k = 10,
+ .m = 10,
+ .w = 16,
+ .hd = 11,
+ .ct = CHKSUM_NONE,
+};
+
+struct ec_args *internal_rs_vand_test_args[] = { &internal_rs_vand_args,
+ &internal_rs_vand_44_args,
+ &internal_rs_vand_1010_args,
+ &internal_rs_vand_48_args,
+ NULL };
+
struct ec_args **all_backend_tests[] = { null_test_args,
flat_xor_test_args,
jerasure_rs_vand_test_args,
jerasure_rs_cauchy_test_args,
isa_l_test_args,
- shss_test_args , NULL};
+ shss_test_args,
+ internal_rs_vand_test_args,
+ NULL};
int num_backends()
{
@@ -237,6 +278,8 @@ char * get_name_from_backend_id(ec_backend_id_t be) {
return ISA_L_RS_VAND_BACKEND;
case EC_BACKEND_SHSS:
return SHSS_BACKEND;
+ case EC_BACKEND_INTERNAL_RS_VAND:
+ return RS_VAND_BACKEND;
default:
return "UNKNOWN";
}
@@ -259,6 +302,9 @@ struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct, int ba
case EC_BACKEND_JERASURE_RS_CAUCHY:
backend_args_array = jerasure_rs_cauchy_test_args;
break;
+ case EC_BACKEND_INTERNAL_RS_VAND:
+ backend_args_array = internal_rs_vand_test_args;
+ break;
case EC_BACKEND_FLAT_XOR_HD:
backend_args_array = flat_xor_test_args;
break;
@@ -1782,6 +1828,67 @@ struct testcase testcases[] = {
test_verify_stripe_metadata_be_ver_mismatch,
EC_BACKEND_SHSS, CHKSUM_CRC32,
.skip = false},
+ // Internal RS Vand backend tests
+ {"create_and_destroy_backend",
+ test_create_and_destroy_backend,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"simple_encode_internal_rs_vand",
+ test_simple_encode_decode,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"decode_with_missing_data_internal_rs_vand",
+ test_decode_with_missing_data,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"decode_with_missing_multi_data_internal_rs_vand",
+ test_decode_with_missing_multi_data,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"decode_with_missing_multi_parity_internal_rs_vand",
+ test_decode_with_missing_multi_parity,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"test_decode_with_missing_multi_data_parity_internal_rs_vand",
+ test_decode_with_missing_multi_data_parity,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"simple_reconstruct_internal_rs_vand",
+ test_simple_reconstruct,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"test_fragments_needed_internal_rs_vand",
+ test_fragments_needed,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"test_get_fragment_metadata_internal_rs_vand",
+ test_get_fragment_metadata,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+ .skip = false},
+ {"test_get_fragment_metadata_internal_rs_vand_crc32",
+ test_get_fragment_metadata,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+ .skip = false},
+ {"test_verify_stripe_metadata",
+ test_verify_stripe_metadata,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+ .skip = false},
+ {"test_verify_stripe_metadata_libec_mismatch",
+ test_verify_stripe_metadata_libec_mismatch,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+ .skip = false},
+ {"test_verify_stripe_metadata_magic_mismatch",
+ test_verify_stripe_metadata_magic_mismatch,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+ .skip = false},
+ {"test_verify_stripe_metadata_be_id_mismatch",
+ test_verify_stripe_metadata_be_id_mismatch,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+ .skip = false},
+ {"test_verify_stripe_metadata_be_ver_mismatch",
+ test_verify_stripe_metadata_be_ver_mismatch,
+ EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+ .skip = false},
{ NULL, NULL, 0, 0, false },
};