Plugging new internal RS backend into liberasurecode.internal_rs

5 files changed, 430 insertions, 7 deletions

diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h
index 4604c2a..e40c164 100644
--- a/include/erasurecode/erasurecode.h
+++ b/include/erasurecode/erasurecode.h
@@ -40,12 +40,13 @@ extern "C" {
 /* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */
 
 typedef enum {
-    EC_BACKEND_NULL                 = 0,
-    EC_BACKEND_JERASURE_RS_VAND     = 1,
-    EC_BACKEND_JERASURE_RS_CAUCHY   = 2,
-    EC_BACKEND_FLAT_XOR_HD          = 3,
-    EC_BACKEND_ISA_L_RS_VAND        = 4,
-    EC_BACKEND_SHSS                 = 5,
+    EC_BACKEND_NULL                   = 0,
+    EC_BACKEND_JERASURE_RS_VAND       = 1,
+    EC_BACKEND_JERASURE_RS_CAUCHY     = 2,
+    EC_BACKEND_FLAT_XOR_HD            = 3,
+    EC_BACKEND_ISA_L_RS_VAND          = 4,
+    EC_BACKEND_SHSS                   = 5,
+    EC_BACKEND_INTERNAL_RS_VAND       = 6,
     EC_BACKENDS_MAX,
 } ec_backend_id_t;
 
diff --git a/src/Makefile.am b/src/Makefile.am
index 1a837c6..63b2542 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -21,6 +21,7 @@ liberasurecode_la_SOURCES = \
 		backends/jerasure/jerasure_rs_vand.c \
 		backends/jerasure/jerasure_rs_cauchy.c \
 		backends/isa-l/isa_l_rs_vand.c \
+    backends/rs_vand/rs_vand.c \
 		backends/shss/shss.c
 
 # Install additional header files
diff --git a/src/backends/rs_vand/rs_vand.c b/src/backends/rs_vand/rs_vand.c
new file mode 100644
index 0000000..b709e13
--- /dev/null
+++ b/src/backends/rs_vand/rs_vand.c
@@ -0,0 +1,312 @@
+/* 
+ * Copyright 2015 Kevin M Greenan
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this
+ * list of conditions and the following disclaimer in the documentation and/or
+ * other materials provided with the distribution.  THIS SOFTWARE IS PROVIDED BY
+ * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * vi: set noai tw=79 ts=4 sw=4:
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "erasurecode.h"
+#include "erasurecode_backend.h"
+#include "erasurecode_helpers.h"
+
+#define INTERNAL_RS_VAND_LIB_MAJOR 1
+#define INTERNAL_RS_VAND_LIB_MINOR 0
+#define INTERNAL_RS_VAND_LIB_REV   0
+#define INTERNAL_RS_VAND_LIB_VER_STR "1.0"
+#define INTERNAL_RS_VAND_LIB_NAME "liberasurecode_rsvand"
+#if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__)
+#define INTERNAL_RS_VAND_SO_NAME "liberasurecode_rsvand.dylib"
+#else
+#define INTERNAL_RS_VAND_SO_NAME "liberasurecode_rsvand.so"
+#endif
+
+/* Forward declarations */
+struct ec_backend_op_stubs internal_rs_vand_ops;
+struct ec_backend internal_rs_vand;
+struct ec_backend_common backend_internal_rs_vand;
+
+typedef int (*internal_rs_vand_encode_func)(int *, char **, char **, int, int, int);
+typedef int (*internal_rs_vand_decode_func)(int *, char **, char **, int, int, int *, int, int);
+typedef int (*internal_rs_vand_reconstruct_func)(int *, char **, char **, int, int, int *, int, int);
+typedef void (*init_rs_vand_func)(int, int);
+typedef void (*deinit_rs_vand_func)();
+typedef void (*free_systematic_matrix_func)(int *);
+typedef int* (*make_systematic_matrix_func)(int, int);
+
+
+struct internal_rs_vand_descriptor {
+    /* calls required for init */
+    init_rs_vand_func init_rs_vand; 
+    deinit_rs_vand_func deinit_rs_vand;
+    free_systematic_matrix_func free_systematic_matrix;
+    make_systematic_matrix_func make_systematic_matrix;
+ 
+    /* calls required for encode */
+    internal_rs_vand_encode_func internal_rs_vand_encode;
+     
+    /* calls required for decode */
+    internal_rs_vand_decode_func internal_rs_vand_decode; 
+    
+    /* calls required for reconstruct */
+    internal_rs_vand_reconstruct_func internal_rs_vand_reconstruct;
+
+    /* fields needed to hold state */
+    int *matrix;
+    int k;
+    int m;
+    int w;
+};
+
+static int internal_rs_vand_encode(void *desc, char **data, char **parity,
+        int blocksize)
+{
+    struct internal_rs_vand_descriptor *rs_vand_desc = 
+        (struct internal_rs_vand_descriptor*) desc;
+
+    /* FIXME: Should this return something? */
+    rs_vand_desc->internal_rs_vand_encode(rs_vand_desc->matrix, data, parity,
+        rs_vand_desc->k, rs_vand_desc->m, blocksize);
+    return 0;
+}
+
+static int internal_rs_vand_decode(void *desc, char **data, char **parity,
+        int *missing_idxs, int blocksize)
+{
+    struct internal_rs_vand_descriptor *rs_vand_desc = 
+        (struct internal_rs_vand_descriptor*) desc;
+
+    /* FIXME: Should this return something? */
+    rs_vand_desc->internal_rs_vand_decode(rs_vand_desc->matrix, data, parity, 
+        rs_vand_desc->k, rs_vand_desc->m, missing_idxs, blocksize, 1);
+
+    return 0;
+}
+
+static int internal_rs_vand_reconstruct(void *desc, char **data, char **parity,
+        int *missing_idxs, int destination_idx, int blocksize)
+{
+    struct internal_rs_vand_descriptor *rs_vand_desc = 
+        (struct internal_rs_vand_descriptor*) desc;
+
+    /* FIXME: Should this return something? */
+    rs_vand_desc->internal_rs_vand_reconstruct(rs_vand_desc->matrix, data, parity, 
+        rs_vand_desc->k, rs_vand_desc->m, missing_idxs, destination_idx, blocksize);
+
+    return 0;
+}
+
+static int internal_rs_vand_min_fragments(void *desc, int *missing_idxs,
+        int *fragments_to_exclude, int *fragments_needed)
+{
+    struct internal_rs_vand_descriptor *rs_vand_desc = 
+        (struct internal_rs_vand_descriptor*)desc;
+
+    uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
+    uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
+    int i;
+    int j = 0;
+    int ret = -1;
+
+    for (i = 0; i < (rs_vand_desc->k + rs_vand_desc->m); i++) {
+        if (!(missing_bm & (1 << i))) {
+            fragments_needed[j] = i;
+            j++;
+        }
+        if (j == rs_vand_desc->k) {
+            ret = 0;
+            fragments_needed[j] = -1;
+            break;
+        }
+    }
+
+    return ret;
+}
+
+static void * internal_rs_vand_init(struct ec_backend_args *args,
+        void *backend_sohandle)
+{
+    struct internal_rs_vand_descriptor *desc = NULL;
+    
+    desc = (struct internal_rs_vand_descriptor *)
+           malloc(sizeof(struct internal_rs_vand_descriptor));
+    if (NULL == desc) {
+        return NULL;
+    }
+
+    desc->k = args->uargs.k;
+    desc->m = args->uargs.m;
+
+    /* store w back in args so upper layer can get to it */
+    desc->w = 16; // w is currently hard-coded at 16
+
+    // This check should not matter, since 64K is way higher
+    // than anyone should ever use
+    if ((desc->k + desc->m) > 65536) {
+        goto error;
+    }
+
+     /*
+     * ISO C forbids casting a void* to a function pointer.
+     * Since dlsym return returns a void*, we use this union to
+     * "transform" the void* to a function pointer.
+     */
+    union {
+        init_rs_vand_func initp;
+        deinit_rs_vand_func deinitp;
+        free_systematic_matrix_func freematrixp;
+        make_systematic_matrix_func makematrixp;
+        internal_rs_vand_encode_func encodep;
+        internal_rs_vand_decode_func decodep;
+        internal_rs_vand_reconstruct_func reconstructp;
+        void *vptr;
+    } func_handle = {.vptr = NULL};
+
+
+    /* fill in function addresses */
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "init_rs_vand");
+    desc->init_rs_vand = func_handle.initp;
+    if (NULL == desc->init_rs_vand) {
+        goto error; 
+    }
+    
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "deinit_rs_vand");
+    desc->deinit_rs_vand = func_handle.deinitp;
+    if (NULL == desc->deinit_rs_vand) {
+        goto error; 
+    }
+    
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "make_systematic_matrix");
+    desc->make_systematic_matrix = func_handle.makematrixp;
+    if (NULL == desc->make_systematic_matrix) {
+        goto error; 
+    }
+    
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "free_systematic_matrix");
+    desc->free_systematic_matrix = func_handle.freematrixp;
+    if (NULL == desc->free_systematic_matrix) {
+        goto error; 
+    }
+    
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "internal_rs_vand_encode");
+    desc->internal_rs_vand_encode = func_handle.encodep;
+    if (NULL == desc->internal_rs_vand_encode) {
+        goto error; 
+    }
+    
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "internal_rs_vand_decode");
+    desc->internal_rs_vand_decode = func_handle.decodep;
+    if (NULL == desc->internal_rs_vand_decode) {
+        goto error; 
+    }
+    
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "internal_rs_vand_reconstruct");
+    desc->internal_rs_vand_reconstruct = func_handle.reconstructp;
+    if (NULL == desc->internal_rs_vand_reconstruct) {
+        goto error; 
+    }
+  
+    desc->init_rs_vand(desc->k, desc->m);
+
+    desc->matrix = desc->make_systematic_matrix(desc->k, desc->m);
+            
+    if (NULL == desc->matrix) {
+        goto error; 
+    }
+
+    return desc;
+
+error:
+    free(desc);
+    
+    return NULL;
+}
+
+/**
+ * Return the element-size, which is the number of bits stored 
+ * on a given device, per codeword.  For Vandermonde, this is 
+ * 'w'.  For somthing like cauchy, this is packetsize * w. 
+ * 
+ * Returns the size in bits!
+ */
+static int
+internal_rs_vand_element_size(void* desc)
+{
+    struct internal_rs_vand_descriptor *rs_vand_desc = NULL;
+    
+    rs_vand_desc = (struct internal_rs_vand_descriptor*) desc;
+
+    return rs_vand_desc->w;
+}
+
+static int internal_rs_vand_exit(void *desc)
+{
+    struct internal_rs_vand_descriptor *rs_vand_desc = NULL;
+    
+    rs_vand_desc = (struct internal_rs_vand_descriptor*) desc;
+
+    rs_vand_desc->free_systematic_matrix(rs_vand_desc->matrix);
+    rs_vand_desc->deinit_rs_vand();
+    free(rs_vand_desc);
+
+    return 0;
+}
+
+/*
+ * For the time being, we only claim compatibility with versions that
+ * match exactly
+ */
+static bool internal_rs_vand_is_compatible_with(uint32_t version) {
+    return version == backend_internal_rs_vand.ec_backend_version;
+}
+
+struct ec_backend_op_stubs internal_rs_vand_op_stubs = {
+    .INIT                       = internal_rs_vand_init,
+    .EXIT                       = internal_rs_vand_exit,
+    .ENCODE                     = internal_rs_vand_encode,
+    .DECODE                     = internal_rs_vand_decode,
+    .FRAGSNEEDED                = internal_rs_vand_min_fragments,
+    .RECONSTRUCT                = internal_rs_vand_reconstruct,
+    .ELEMENTSIZE                = internal_rs_vand_element_size,
+    .ISCOMPATIBLEWITH           = internal_rs_vand_is_compatible_with,
+};
+
+struct ec_backend_common backend_internal_rs_vand = {
+    .id                         = EC_BACKEND_INTERNAL_RS_VAND,
+    .name                       = INTERNAL_RS_VAND_LIB_NAME,
+    .soname                     = INTERNAL_RS_VAND_SO_NAME,
+    .soversion                  = INTERNAL_RS_VAND_LIB_VER_STR,
+    .ops                        = &internal_rs_vand_op_stubs,
+    .backend_metadata_size      = 0,
+    .ec_backend_version         = _VERSION(INTERNAL_RS_VAND_LIB_MAJOR,
+                                           INTERNAL_RS_VAND_LIB_MINOR,
+                                           INTERNAL_RS_VAND_LIB_REV),
+};
diff --git a/src/erasurecode.c b/src/erasurecode.c
index d9a868c..3c9e123 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -44,6 +44,7 @@ extern struct ec_backend_common backend_jerasure_rs_vand;
 extern struct ec_backend_common backend_jerasure_rs_cauchy;
 extern struct ec_backend_common backend_isa_l_rs_vand;
 extern struct ec_backend_common backend_shss;
+extern struct ec_backend_common backend_internal_rs_vand;
 
 ec_backend_t ec_backends_supported[] = {
     (ec_backend_t) &backend_null,
@@ -52,6 +53,7 @@ ec_backend_t ec_backends_supported[] = {
     (ec_backend_t) &backend_flat_xor_hd,
     (ec_backend_t) &backend_isa_l_rs_vand,
     (ec_backend_t) &backend_shss,
+    (ec_backend_t) &backend_internal_rs_vand,
     NULL,
 };
 
diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c
index c9d67af..9f1ad9d 100644
--- a/test/liberasurecode_test.c
+++ b/test/liberasurecode_test.c
@@ -39,6 +39,7 @@
 #define JERASURE_RS_CAUCHY_BACKEND "jerasure_rs_cauchy"
 #define ISA_L_RS_VAND_BACKEND "isa_l_rs_vand"
 #define SHSS_BACKEND "shss"
+#define RS_VAND_BACKEND "rs_vand"
 
 typedef void (*TEST_FUNC)();
 
@@ -180,12 +181,52 @@ struct ec_args shss_args = {
 
 struct ec_args *shss_test_args[] = { &shss_args, NULL };
 
+struct ec_args internal_rs_vand_args = {
+    .k = 10,
+    .m = 4,
+    .w = 16,
+    .hd = 5,
+    .ct = CHKSUM_NONE,
+};
+
+struct ec_args internal_rs_vand_44_args = {
+    .k = 4,
+    .m = 4,
+    .w = 16,
+    .hd = 5,
+    .ct = CHKSUM_NONE,
+};
+
+struct ec_args internal_rs_vand_48_args = {
+    .k = 4,
+    .m = 8,
+    .w = 16,
+    .hd = 9,
+    .ct = CHKSUM_NONE,
+};
+
+struct ec_args internal_rs_vand_1010_args = {
+    .k = 10,
+    .m = 10,
+    .w = 16,
+    .hd = 11,
+    .ct = CHKSUM_NONE,
+};
+
+struct ec_args *internal_rs_vand_test_args[] = { &internal_rs_vand_args, 
+                                                 &internal_rs_vand_44_args, 
+                                                 &internal_rs_vand_1010_args, 
+                                                 &internal_rs_vand_48_args, 
+                                                 NULL };
+
 struct ec_args **all_backend_tests[] = {  null_test_args, 
                                           flat_xor_test_args, 
                                           jerasure_rs_vand_test_args, 
                                           jerasure_rs_cauchy_test_args, 
                                           isa_l_test_args, 
-                                          shss_test_args , NULL};
+                                          shss_test_args, 
+                                          internal_rs_vand_test_args,
+                                          NULL};
 
 int num_backends()
 {
@@ -237,6 +278,8 @@ char * get_name_from_backend_id(ec_backend_id_t be) {
             return ISA_L_RS_VAND_BACKEND;
         case EC_BACKEND_SHSS:
             return SHSS_BACKEND;
+        case EC_BACKEND_INTERNAL_RS_VAND:
+            return RS_VAND_BACKEND;
         default:
             return "UNKNOWN";
     }
@@ -259,6 +302,9 @@ struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct, int ba
         case EC_BACKEND_JERASURE_RS_CAUCHY:
             backend_args_array = jerasure_rs_cauchy_test_args;
             break;
+        case EC_BACKEND_INTERNAL_RS_VAND:
+            backend_args_array = internal_rs_vand_test_args;
+            break;
         case EC_BACKEND_FLAT_XOR_HD:
             backend_args_array = flat_xor_test_args;
             break;
@@ -1782,6 +1828,67 @@ struct testcase testcases[] = {
         test_verify_stripe_metadata_be_ver_mismatch,
         EC_BACKEND_SHSS, CHKSUM_CRC32,
         .skip = false},
+    // Internal RS Vand backend tests
+    {"create_and_destroy_backend",
+        test_create_and_destroy_backend,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"simple_encode_internal_rs_vand",
+        test_simple_encode_decode,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"decode_with_missing_data_internal_rs_vand",
+        test_decode_with_missing_data,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"decode_with_missing_multi_data_internal_rs_vand",
+        test_decode_with_missing_multi_data,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"decode_with_missing_multi_parity_internal_rs_vand",
+        test_decode_with_missing_multi_parity,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"test_decode_with_missing_multi_data_parity_internal_rs_vand",
+        test_decode_with_missing_multi_data_parity,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"simple_reconstruct_internal_rs_vand",
+        test_simple_reconstruct,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"test_fragments_needed_internal_rs_vand",
+        test_fragments_needed,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"test_get_fragment_metadata_internal_rs_vand",
+        test_get_fragment_metadata,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_NONE,
+        .skip = false},
+    {"test_get_fragment_metadata_internal_rs_vand_crc32",
+        test_get_fragment_metadata,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata",
+        test_verify_stripe_metadata,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_libec_mismatch",
+         test_verify_stripe_metadata_libec_mismatch,
+         EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+         .skip = false},
+    {"test_verify_stripe_metadata_magic_mismatch",
+          test_verify_stripe_metadata_magic_mismatch,
+          EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+         .skip = false},
+    {"test_verify_stripe_metadata_be_id_mismatch",
+         test_verify_stripe_metadata_be_id_mismatch,
+         EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+         .skip = false},
+    {"test_verify_stripe_metadata_be_ver_mismatch",
+        test_verify_stripe_metadata_be_ver_mismatch,
+        EC_BACKEND_INTERNAL_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
     { NULL, NULL, 0, 0, false },
 };