From 53b0ea706358f87b1c7fa63659b3b2799448f510 Mon Sep 17 00:00:00 2001 From: "Disch, Simon" Date: Tue, 24 Feb 2015 15:03:45 +0100 Subject: latest version of key-value-store (updated database structure, using mmap for database access, added recovery mechanism) --- inc/protected/persComDbAccess.h | 11 +- inc/protected/persComErrors.h | 2 + src/key-value-store/crc32.c | 47 +- src/key-value-store/database/kissdb.c | 2791 +++++++++++++-------- src/key-value-store/database/kissdb.h | 176 +- src/key-value-store/hashtable/qhash.h | 7 - src/key-value-store/hashtable/qhasharr.c | 132 +- src/key-value-store/hashtable/qhasharr.h | 4 +- src/key-value-store/pers_low_level_db_access.c | 1637 ++++++++----- src/pers_local_shared_db_access.c | 12 + test/Makefile.am | 9 +- test/test_pco_key_value_store.c | 3135 ++++++++++++++++++++++++ 12 files changed, 6124 insertions(+), 1839 deletions(-) create mode 100644 test/test_pco_key_value_store.c diff --git a/inc/protected/persComDbAccess.h b/inc/protected/persComDbAccess.h index 0a169fd..2b79640 100644 --- a/inc/protected/persComDbAccess.h +++ b/inc/protected/persComDbAccess.h @@ -44,14 +44,21 @@ extern "C" */ /* maximum data size for a key type resourceID */ #define PERS_DB_MAX_LENGTH_KEY_NAME 128 /**< Max. length of the key identifier */ -#define PERS_DB_MAX_SIZE_KEY_DATA 16384 /**< Max. size of the key entry (slot definition) */ -/** \} */ +#define PERS_DB_MAX_SIZE_KEY_DATA 8028 /**< Max. size of the key entry (slot definition) */ +/** \} */ /** \defgroup PERS_DB_ACCESS_FUNCTIONS Functions * \{ */ + /** + * \brief returns the max DB key data size + * + * \return the size + */ +int persComDbgetMaxKeyValueSize(void); + /** * \brief Obtain a handler to DB indicated by dbPathname diff --git a/inc/protected/persComErrors.h b/inc/protected/persComErrors.h index ca447bb..34bf729 100644 --- a/inc/protected/persComErrors.h +++ b/inc/protected/persComErrors.h @@ -55,6 +55,8 @@ extern "C" #define PERS_COM_ERR_ACCESS_DENIED (PERS_COM_ERROR_CODE - 7) //!< Insufficient rights to perform opperation #define PERS_COM_ERR_OUT_OF_MEMORY (PERS_COM_ERROR_CODE - 8) //!< Not enough resources for an opperation +#define PERS_COM_ERR_READONLY (PERS_COM_ERROR_CODE - 9) //!< Database was opened in readonly mode and cannot be written + /* IPC specific error codes */ #define PERS_COM_IPC_ERR_PCL_NOT_AVAILABLE (PERS_COM_ERROR_CODE - 255) //!< PCL client not available (application was killed) /* end of IPC specific error codes */ diff --git a/src/key-value-store/crc32.c b/src/key-value-store/crc32.c index caa58e4..55c9d08 100644 --- a/src/key-value-store/crc32.c +++ b/src/key-value-store/crc32.c @@ -118,53 +118,16 @@ unsigned int pcoCrc32(unsigned int crc, const unsigned char *buf, size_t theSize p = buf; crc = crc ^ ~0U; - if(p != 0) + if (p != 0) { - while(theSize--) + while (theSize--) { - unsigned int idx = (crc ^ *p++) & 0xFF; + unsigned int idx = (crc ^ *p++) & 0xFF; - if(idx < crc32_array_size) - crc = crc32_tab[idx] ^ (crc >> 8); + if (idx < crc32_array_size) + crc = crc32_tab[idx] ^ (crc >> 8); } rval = crc ^ ~0U; } return rval; } - - -int pcoCalcCrc32Csum(int fd, int startOffset) -{ - int rval = 1; - char* buf; - struct stat statBuf; - unsigned int crc = 0; - - fstat(fd, &statBuf); - buf = malloc((unsigned int) statBuf.st_size- startOffset); - - if (buf != 0) - { - memset(buf, 0, statBuf.st_size- startOffset); - off_t curPos = 0; - // remember the current position - curPos = lseek(fd, 0, SEEK_CUR); - // set to start offset - lseek(fd, startOffset, SEEK_SET); - //printf("FSTAT -> Filesize: %d \n", (int) statBuf.st_size); - if( read(fd, buf, statBuf.st_size- startOffset) != statBuf.st_size- startOffset) - return -1; - crc = 0; - crc = pcoCrc32(crc, (unsigned char*) buf, statBuf.st_size- startOffset); - rval = crc; - // set back to the position - lseek(fd, curPos, SEEK_SET); - free(buf); - } - else - rval = -1; - return rval; -} - - - diff --git a/src/key-value-store/database/kissdb.c b/src/key-value-store/database/kissdb.c index 4c8e7b6..6a4d119 100644 --- a/src/key-value-store/database/kissdb.c +++ b/src/key-value-store/database/kissdb.c @@ -1,8 +1,8 @@ - /****************************************************************************** - * Project Persistency - * (c) copyright 2014 - * Company XS Embedded GmbH - *****************************************************************************/ +/****************************************************************************** +* Project Persistency +* (c) copyright 2014 +* Company XS Embedded GmbH +*****************************************************************************/ /* (Keep It) Simple Stupid Database * * Written by Adam Ierymenko @@ -17,14 +17,8 @@ /* Note: big-endian systems will need changes to implement byte swapping * on hash table file I/O. Or you could just use it as-is if you don't care * that your database files will be unreadable on little-endian systems. */ - #define _FILE_OFFSET_BITS 64 -#define TMP_BUFFER_LENGTH 128 #define KISSDB_HEADER_SIZE sizeof(Header_s) -#define __useBackups -//#define __useFileMapping -//#define __writeThrough -#define __checkerror #include "./kissdb.h" #include "../crc32.h" @@ -37,11 +31,20 @@ #include #include #include +#include #include +#include +#include +#include "persComErrors.h" -#include "dlt.h" +// +//#ifdef COND_GCOV +//extern void __gcov_flush(void); +//#endif -DLT_DECLARE_CONTEXT(persComLldbDLTCtx); +//#define PFS_TEST +//extern DltContext persComLldbDLTCtx; +DLT_IMPORT_CONTEXT (persComLldbDLTCtx) #ifdef __showTimeMeasurements inline long long getNsDuration(struct timespec* start, struct timespec* end) @@ -51,43 +54,59 @@ inline long long getNsDuration(struct timespec* start, struct timespec* end) #endif /* djb2 hash function */ -static uint64_t KISSDB_hash(const void *b, unsigned long len) +static uint64_t KISSDB_hash(const void* b, unsigned long len) { unsigned long i; uint64_t hash = 5381; for (i = 0; i < len; ++i) - hash = ((hash << 5) + hash) + (uint64_t) (((const uint8_t *) b)[i]); + { + hash = ((hash << 5) + hash) + (uint64_t) (((const uint8_t*) b)[i]); + } return hash; } +#if 1 //returns a name for shared memory objects beginning with a slash followed by "path" (non alphanumeric chars are replaced with '_') appended with "tailing" -char * kdbGetShmName(const char *tailing, const char * path) +char* kdbGetShmName(const char* tailing, const char* path) { - char * result = (char *) malloc(1 + strlen(path) + strlen(tailing) + 1); //free happens at lifecycle shutdown + int pathLen = strlen(path); + int tailLen = strlen(tailing); + char* result = (char*) malloc(1 + pathLen + tailLen + 1); //free happens at lifecycle shutdown int i =0; int x = 1; if (result != NULL) { result[0] = '/'; - for (i = 0; i < strlen(path); i++) + for (i = 0; i < pathLen; i++) { if (!isalnum(path[i])) + { result[i + 1] = '_'; + } else + { result[i + 1] = path[i]; + } } - for (x = 0; x < strlen(tailing); x++) + for (x = 0; x < tailLen; x++) { result[i + x + 1] = tailing[x]; } result[i + x + 1] = '\0'; } + else + { + result = NULL; + } return result; } +#endif + + //returns -1 on error and positive value for success -int kdbShmemOpen(const char * name, size_t length, Kdb_bool* shmCreator) +int kdbShmemOpen(const char* name, size_t length, Kdb_bool* shmCreator) { int result; result = shm_open(name, O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR); @@ -98,111 +117,268 @@ int kdbShmemOpen(const char * name, size_t length, Kdb_bool* shmCreator) *shmCreator = Kdb_false; result = shm_open(name, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); if (result < 0) + { return -1; + } } } else { *shmCreator = Kdb_true; if (ftruncate(result, length) < 0) + { return -1; + } } return result; } -void Kdb_wrlock(pthread_rwlock_t * wrlock) +void Kdb_wrlock(pthread_rwlock_t* wrlock) { pthread_rwlock_wrlock(wrlock); } -void Kdb_rdlock(pthread_rwlock_t * rdlock) -{ - pthread_rwlock_rdlock(rdlock); -} +//void Kdb_rdlock(pthread_rwlock_t* rdlock) +//{ +// pthread_rwlock_rdlock(rdlock); +//} -void Kdb_unlock(pthread_rwlock_t * lock) +void Kdb_unlock(pthread_rwlock_t* lock) { pthread_rwlock_unlock(lock); } -Kdb_bool kdbShmemClose(int shmem, const char * shmName) +Kdb_bool kdbShmemClose(int shmem, const char* shmName) { if( close(shmem) == -1) + { return Kdb_false; + } if( shm_unlink(shmName) < 0) + { return Kdb_false; + } return Kdb_true; } -void * getKdbShmemPtr(int shmem, size_t length) +void* getKdbShmemPtr(int shmem, size_t length) { void* result = mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, shmem, 0); if (result == MAP_FAILED) - return ((void *) -1); + { + return ((void*) -1); + } return result; } -Kdb_bool freeKdbShmemPtr(void * shmem_ptr, size_t length) + +Kdb_bool freeKdbShmemPtr(void* shmem_ptr, size_t length) { if(munmap(shmem_ptr, length) == 0) + { return Kdb_true; + } else + { return Kdb_false; + } } -Kdb_bool resizeKdbShmem(int shmem, Hashtable_slot_s** shmem_ptr, size_t oldLength, size_t newLength) +Kdb_bool resizeKdbShmem(int shmem, Hashtable_s** shmem_ptr, size_t oldLength, size_t newLength) { //unmap shm with old size if( freeKdbShmemPtr(*shmem_ptr, oldLength) == Kdb_false) + { return Kdb_false; + } if (ftruncate(shmem, newLength) < 0) + { return Kdb_false; + } //get pointer to resized shm with new Length *shmem_ptr = getKdbShmemPtr(shmem, newLength); - if(*shmem_ptr == ((void *) -1)) + if(*shmem_ptr == ((void*) -1)) + { return Kdb_false; + } return Kdb_true; } -#ifdef __writeThrough -Kdb_bool remapKdbShmem(int shmem, uint64_t** shmem_ptr, size_t oldLength, size_t newLength) + +Kdb_bool remapSharedHashtable(int shmem, Hashtable_s** shmem_ptr, size_t oldLength, size_t newLength ) { - //unmap shm with old size - if( freeKdbShmemPtr(*shmem_ptr, oldLength) == Kdb_false ) + //unmap hashtable with old size + if( freeKdbShmemPtr(*shmem_ptr, oldLength) == Kdb_false) + { return Kdb_false; + } //get pointer to resized shm with new Length *shmem_ptr = getKdbShmemPtr(shmem, newLength); - if(*shmem_ptr == ((void *) -1)) + if(*shmem_ptr == ((void*) -1)) + { return Kdb_false; + } return Kdb_true; } + +#if 0 +void printKdb(KISSDB* db) +{ + printf("START ############################### \n"); + printf("db->htSize: %d \n", db->htSize); + printf("db->cacheReferenced: %d \n", db->cacheReferenced); + printf("db->keySize: %" PRId64 " \n", db->keySize); + printf("db->valSize: %" PRId64 " \n", db->valSize); + printf("db->htSizeBytes: %" PRId64 " \n", db->htSizeBytes); + printf("db->htMappedSize: %" PRId64 " \n", db->htMappedSize); + printf("db->dbMappedSize: %" PRId64 " \n", db->dbMappedSize); + printf("db->shmCreator: %d \n", db->shmCreator); + printf("db->alreadyOpen: %d \n", db->alreadyOpen); + printf("db->hashTables: %p \n", db->hashTables); + printf("db->mappedDb: %p \n", db->mappedDb); + printf("db->sharedCache: %p \n", db->sharedCache); + printf("db->sharedFd: %d \n", db->sharedFd); + printf("db->htFd: %d \n", db->htFd); + printf("db->sharedCacheFd: %d \n", db->sharedCacheFd); + printf("db->semName: %s \n", db->semName); + printf("db->sharedName: %s \n", db->sharedName); + printf("db->cacheName: %s \n", db->cacheName); + printf("db->htName: %s \n", db->htName); + printf("db->shared: %p \n", db->shared); + printf("db->tbl: %p \n", db->tbl[0]); + printf("db->kdbSem: %p \n", db->kdbSem); + printf("db->fd: %d \n", db->fd); + printf("END ############################### \n"); +} #endif -int KISSDB_open(KISSDB *db, const char *path, int mode, uint16_t hash_table_size, uint64_t key_size, - uint64_t value_size) +int KISSDB_open(KISSDB* db, const char* path, int openMode, int writeMode, uint16_t hash_table_size, uint64_t key_size, uint64_t value_size) { - Hashtable_slot_s *httmp; - Kdb_bool tmp_creator; + Hashtable_s* htptr; int ret = 0; + Kdb_bool htFound; + Kdb_bool tmpCreator; + off_t offset = 0; + size_t firstMappSize; + struct stat sb; + + if (db->alreadyOpen == Kdb_false) //check if this instance has already opened the db before + { + db->sharedName = kdbGetShmName("-shm-info", path); + if (db->sharedName == NULL) + { + return KISSDB_ERROR_MALLOC; + } + db->sharedFd = kdbShmemOpen(db->sharedName, sizeof(Shared_Data_s), &db->shmCreator); + if (db->sharedFd < 0) + { + return KISSDB_ERROR_OPEN_SHM; + } + db->shared = (Shared_Data_s*) getKdbShmemPtr(db->sharedFd, sizeof(Shared_Data_s)); + if (db->shared == ((void*) -1)) + { + return KISSDB_ERROR_MAP_SHM; + } + + db->sharedCacheFd = -1; + db->mappedDb = NULL; + + if (db->shmCreator == Kdb_true) + { + //[Initialize rwlock attributes] + pthread_rwlockattr_t rwlattr; + pthread_rwlockattr_init(&rwlattr); + pthread_rwlockattr_setpshared(&rwlattr, PTHREAD_PROCESS_SHARED); + pthread_rwlock_init(&db->shared->rwlock, &rwlattr); - //TODO check if usage of O_SYNC O_DIRECT flags is needed. If O_SYNC and O_DIrect is specified, no additional fsync calls are needed after fflush - if(mode == KISSDB_OPEN_MODE_RWCREAT) - db->fd = open(path, O_CREAT | O_RDWR , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH ); //gets closed when db->f is closed + Kdb_wrlock(&db->shared->rwlock); + + //init cache filedescriptor, reference counter and hashtable number + db->sharedCacheFd = -1; + db->shared->refCount = 0; + db->shared->htNum = 0; + db->shared->mappedDbSize = 0; + db->shared->writeMode = writeMode; + db->shared->openMode = openMode; + } + else + { + Kdb_wrlock(&db->shared->rwlock); + } + } else - db->fd = open(path, O_RDWR , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH ); //gets closed when db->f is closed + { + Kdb_wrlock(&db->shared->rwlock); + } + switch (db->shared->openMode) + { + case KISSDB_OPEN_MODE_RWCREAT: + { + //create database + db->fd = open(path, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + break; + } + case KISSDB_OPEN_MODE_RDWR: + { + //read / write mode + db->fd = open(path, O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + break; + } + case KISSDB_OPEN_MODE_RDONLY: + { + db->fd = open(path, O_RDONLY); + break; + } + default: + { + break; + } + } if(db->fd == -1) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": Opening database file: <"); DLT_STRING(path); DLT_STRING("> failed: "); DLT_STRING(strerror(errno))); + return KISSDB_ERROR_IO; + } + + if( 0 != fstat(db->fd, &sb)) + { return KISSDB_ERROR_IO; + } + + + /* mmap whole database file if it already exists (else the file is mapped in writeheader()) */ + if (sb.st_size > 0) + { + if(db->shared->openMode != KISSDB_OPEN_MODE_RDONLY ) + { + db->mappedDb = (void*) mmap(NULL, sb.st_size, PROT_WRITE | PROT_READ, MAP_SHARED, db->fd, 0); + } + else + { + db->mappedDb = (void*) mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, db->fd, 0); + } + if (db->mappedDb == MAP_FAILED) + { + return KISSDB_ERROR_IO; + } + else + { + //update mapped size + db->shared->mappedDbSize = (uint64_t)sb.st_size; + db->dbMappedSize = db->shared->mappedDbSize; + } + } - if (lseek(db->fd, 0, SEEK_END) == -1) + offset = sb.st_size; + if (offset == -1) { - close(db->fd); return KISSDB_ERROR_IO; } - if (lseek(db->fd, 0, SEEK_CUR) < KISSDB_HEADER_SIZE) + if ( offset < KISSDB_HEADER_SIZE) { /* write header if not already present */ if ((hash_table_size) && (key_size) && (value_size)) @@ -210,1029 +386,891 @@ int KISSDB_open(KISSDB *db, const char *path, int mode, uint16_t hash_table_size ret = writeHeader(db, &hash_table_size, &key_size, &value_size); if(0 != ret) { - close(db->fd); return ret; } - //Seek behind header - if (lseek(db->fd, KISSDB_HEADER_SIZE, SEEK_SET) == -1) - { - close(db->fd); - return KISSDB_ERROR_IO; - } } else { - close(db->fd); return KISSDB_ERROR_INVALID_PARAMETERS; } } else { - //read existing header + /* read existing header to verify database version */ ret = readHeader(db, &hash_table_size, &key_size, &value_size); if( 0 != ret) - return ret; - - if (lseek(db->fd, KISSDB_HEADER_SIZE, SEEK_SET) == -1) { - close(db->fd); - return KISSDB_ERROR_IO; - } //Seek behind header + return ret; + } } //store non shared db information - db->hash_table_size = hash_table_size; - db->key_size = key_size; - db->value_size = value_size; - db->hash_table_size_bytes = sizeof(Hashtable_slot_s) * (hash_table_size + 1); /* [hash_table_size] == next table */ - - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("Hashtable size in bytes: "), DLT_UINT64(db->hash_table_size_bytes)); + db->htSize = hash_table_size; + db->keySize = key_size; + db->valSize = value_size; + db->htSizeBytes = sizeof(Hashtable_s); - if (db->already_open == Kdb_false) //check if this instance has already opened the db before + if (db->alreadyOpen == Kdb_false) //check if this instance has already opened the db before { - db->shmem_cached_name = kdbGetShmName("-cache", path); - if(db->shmem_cached_name == NULL) - return KISSDB_ERROR_MALLOC; - db->shmem_info_name = kdbGetShmName("-shm-info", path); - if(db->shmem_info_name == NULL) + db->cacheName = kdbGetShmName("-cache", path); + if(db->cacheName == NULL) + { return KISSDB_ERROR_MALLOC; - db->shmem_info_fd = kdbShmemOpen(db->shmem_info_name, sizeof(Shared_Data_s), &db->shmem_creator); - if(db->shmem_info_fd < 0) - return KISSDB_ERROR_OPEN_SHM; - db->shmem_info = (Shared_Data_s *) getKdbShmemPtr(db->shmem_info_fd, sizeof(Shared_Data_s)); - if(db->shmem_info == ((void *) -1)) - return KISSDB_ERROR_MAP_SHM; - - size_t first_mapping; - if(db->shmem_info->shmem_size > db->hash_table_size_bytes ) - first_mapping = db->shmem_info->shmem_size; + } + //check if more than one hashtable is already in shared memory + if(db->shared->htShmSize > db->htSizeBytes ) + { + firstMappSize = db->shared->htShmSize; + } else - first_mapping = db->hash_table_size_bytes; + { + firstMappSize = db->htSizeBytes; + } //open / create shared memory for first hashtable - db->shmem_ht_name = kdbGetShmName("-ht", path); - if(db->shmem_ht_name == NULL) + db->htName = kdbGetShmName("-ht", path); + if(db->htName == NULL) + { return KISSDB_ERROR_MALLOC; - db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, first_mapping, &tmp_creator); - if(db->shmem_ht_fd < 0) + } + db->htFd = kdbShmemOpen(db->htName, firstMappSize, &tmpCreator); + if(db->htFd < 0) + { return KISSDB_ERROR_OPEN_SHM; - db->hash_tables = (Hashtable_slot_s *) getKdbShmemPtr(db->shmem_ht_fd, first_mapping); - if(db->hash_tables == ((void *) -1)) - return KISSDB_ERROR_MAP_SHM; - db->old_mapped_size = first_mapping; //local information - - //if shared memory for rwlock was opened (created) with this call to KISSDB_open for the first time -> init rwlock - if (db->shmem_creator == Kdb_true) + } + db->hashTables = (Hashtable_s*) getKdbShmemPtr(db->htFd, firstMappSize); + if(db->hashTables == ((void*) -1)) { - //[Initialize rwlock attributes] - pthread_rwlockattr_t rwlattr, cache_rwlattr; - pthread_rwlockattr_init(&rwlattr); - pthread_rwlockattr_init(&cache_rwlattr); - pthread_rwlockattr_setpshared(&rwlattr, PTHREAD_PROCESS_SHARED); - pthread_rwlockattr_setpshared(&cache_rwlattr, PTHREAD_PROCESS_SHARED); - pthread_rwlock_init(&db->shmem_info->rwlock, &rwlattr); - pthread_rwlock_init(&db->shmem_info->cache_rwlock, &cache_rwlattr); - Kdb_wrlock(&db->shmem_info->rwlock); - -#ifdef __checkerror - //CHECK POWERLOSS FLAGS - ret = checkErrorFlags(db); - if (0 != ret) - { - close(db->fd); - Kdb_unlock(&db->shmem_info->rwlock); - return ret; - } -#endif - db->shmem_info->num_hash_tables = 0; + return KISSDB_ERROR_MAP_SHM; } - else // already initialized - Kdb_wrlock(&db->shmem_info->rwlock); - - db->already_open = Kdb_true; + db->htMappedSize = firstMappSize; + db->alreadyOpen = Kdb_true; } - else - Kdb_wrlock(&db->shmem_info->rwlock); - //only read header from file into memory for first caller of KISSDB_open - if (db->shmem_creator == Kdb_true) + /* + * Read hashtables from file into memory ONLY for first caller of KISSDB_open + * Determine number of existing hashtables (db->shared->htNum) * + */ + if (db->shmCreator == Kdb_true ) { - httmp = (Hashtable_slot_s*) malloc(db->hash_table_size_bytes); //read hashtable from file - if (!httmp) - { - close(db->fd); - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_MALLOC; - } - while (read(db->fd, httmp, db->hash_table_size_bytes) == db->hash_table_size_bytes) + uint64_t offset = KISSDB_HEADER_SIZE; + //only read hashtables from file if file is larger than header + hashtable size + if(db->shared->mappedDbSize >= ( KISSDB_HEADER_SIZE + db->htSizeBytes) ) { - Kdb_bool result = Kdb_false; - //if new size would exceed old shared memory size-> allocate additional memory page to shared memory - if (db->hash_table_size_bytes * (db->shmem_info->num_hash_tables + 1) > db->old_mapped_size) + htptr = (Hashtable_s*) ( db->mappedDb + KISSDB_HEADER_SIZE); + htFound = Kdb_true; + while (htFound && offset < db->dbMappedSize ) { - Kdb_bool temp; - if (db->shmem_ht_fd <= 0) + //check for existing start OR end delimiter of hashtable + if (htptr->delimStart == HASHTABLE_START_DELIMITER || htptr->delimEnd == HASHTABLE_END_DELIMITER) { - db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); - if(db->shmem_ht_fd < 0) + //if new size would exceed old shared memory size-> allocate additional memory page to shared memory + Kdb_bool result = Kdb_false; + if ( (db->htSizeBytes * (db->shared->htNum + 1)) > db->htMappedSize) { - free(httmp); - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_OPEN_SHM; + Kdb_bool temp; + if (db->htFd <= 0) + { + db->htFd = kdbShmemOpen(db->htName, db->htMappedSize, &temp); + if(db->htFd < 0) + { + return KISSDB_ERROR_OPEN_SHM; + } + } + result = resizeKdbShmem(db->htFd, &db->hashTables, db->htMappedSize, db->htMappedSize + db->htSizeBytes); + if (result == Kdb_false) + { + return KISSDB_ERROR_RESIZE_SHM; + } + else + { + db->shared->htShmSize = db->htMappedSize + db->htSizeBytes; + db->htMappedSize = db->shared->htShmSize; + } + } + // copy the current hashtable read from file to (htadress + (htsize * htcount)) in memory + memcpy(((uint8_t*) db->hashTables) + (db->htSizeBytes * db->shared->htNum), htptr, db->htSizeBytes); + ++db->shared->htNum; + + //read until all linked hashtables have been read + if (htptr->slots[db->htSize].offsetA ) //if a offset to a further hashtable exists + { + htptr = (Hashtable_s*) (db->mappedDb + htptr->slots[db->htSize].offsetA); //follow link to next hashtable + offset = htptr->slots[db->htSize].offsetA; + } + else //no link to next hashtable or link is invalid + { + htFound = Kdb_false; } } - result = resizeKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size, db->old_mapped_size + db->hash_table_size_bytes); - if (result == Kdb_false) - { - free(httmp); - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_RESIZE_SHM; - } - else + else //delimiters of first hashtable or linked hashtable are invalid { - db->shmem_info->shmem_size = db->old_mapped_size + db->hash_table_size_bytes; - db->old_mapped_size = db->old_mapped_size + db->hash_table_size_bytes; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": hashtable delimiters are invalid -> rebuild not possible!")); + htFound = Kdb_false; } } - // copy the current hashtable read from file to (htadress + (htsize * htcount)) in memory - memcpy(((uint8_t *) db->hash_tables) + (db->hash_table_size_bytes * db->shmem_info->num_hash_tables), httmp, db->hash_table_size_bytes); - ++db->shmem_info->num_hash_tables; - - //read until all hash tables have been read - if (httmp[db->hash_table_size].offsetA) //if httable[hash_table_size] contains a offset to a further hashtable + } + /* + * CHECK POWERLOSS FLAGS AND REBUILD DATABASE IF NECESSARY + */ + if (db->shared->openMode != KISSDB_OPEN_MODE_RDONLY) + { + if (checkErrorFlags(db) != 0) { - //ONE MORE HASHTABLE FOUND - if (lseek(db->fd, httmp[db->hash_table_size].offsetA, SEEK_SET) == -1) - { //move the filepointer to the next hashtable in the file - KISSDB_close(db); - free(httmp); - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(": database was not closed correctly in last lifecycle!")); + if (verifyHashtableCS(db) != 0) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(": A hashtable is invalid -> Start rebuild of hashtables!")); + if (rebuildHashtables(db) != 0) //hashtables are corrupt, walk through the database and search for data blocks -> then rebuild the hashtables + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": hashtable rebuild failed!")); + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(__FUNCTION__); DLT_STRING(": hashtable rebuild successful!")); + } + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":Start datablock check / recovery!")); + recoverDataBlocks(db); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":End datablock check / recovery!")); + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":Start datablock check / recovery!")); + recoverDataBlocks(db); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":End datablock check / recovery!")); } } - else - break; // no further hashtables exist } - free(httmp); } - - //printSharedHashtable(db); - - Kdb_unlock(&db->shmem_info->rwlock); + Kdb_unlock(&db->shared->rwlock); return 0; } - - - -int KISSDB_close(KISSDB *db) +int KISSDB_close(KISSDB* db) { - Kdb_wrlock(&db->shmem_info->rwlock); +#ifdef PFS_TEST + printf(" START: KISSDB_CLOSE \n"); +#endif - uint64_t crc = 0; + Hashtable_s* htptr = NULL; Header_s* ptr = 0; -#ifdef __showTimeMeasurements - long long KdbDuration = 0; - struct timespec mmapStart, mmapEnd; - KdbDuration = 0; -#endif + uint64_t crc = 0; - //printSharedHashtable(db); - if (db->shmem_creator == Kdb_true) + Kdb_wrlock(&db->shared->rwlock); + + //if no other instance has opened the database + if( db->shared->refCount == 0) { - //free shared hashtable - if( freeKdbShmemPtr(db->hash_tables, db->old_mapped_size) == Kdb_false) + if (db->shared->openMode != KISSDB_OPEN_MODE_RDONLY) { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_UNMAP_SHM; + if(db->htMappedSize < db->shared->htShmSize) + { + if ( Kdb_false == remapSharedHashtable(db->htFd, &db->hashTables, db->htMappedSize, db->shared->htShmSize)) + { + return KISSDB_ERROR_RESIZE_SHM; + } + else + { + db->htMappedSize = db->shared->htShmSize; + } + } + //remap database file if in the meanwhile another process added new data (key value pairs / hashtables) to the file (only happens if writethrough is used) + if (db->dbMappedSize < db->shared->mappedDbSize) + { + db->mappedDb = mremap(db->mappedDb, db->dbMappedSize, db->shared->mappedDbSize, MREMAP_MAYMOVE); + if (db->mappedDb == MAP_FAILED) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":mremap error: !"), DLT_STRING(strerror(errno))); + return KISSDB_ERROR_IO; + } + else + { + db->dbMappedSize = db->shared->mappedDbSize; + } + } + + // generate checksum for every hashtable and write crc to file + if (db->fd) + { + int i = 0; + int offset = sizeof(Header_s); //offset in file to first hashtable + if (db->shared->htNum > 0) //if hashtables exist + { + //write hashtables and crc to file + for (i = 0; i < db->shared->htNum; i++) + { + crc = 0; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) db->hashTables[i].slots, sizeof(db->hashTables[i].slots)); + db->hashTables[i].crc = crc; + htptr = (Hashtable_s*) (db->mappedDb + offset); + //copy hashtable and generated crc from shared memory to mapped hashtable in file + memcpy(htptr, &db->hashTables[i], db->htSizeBytes); + offset = db->hashTables[i].slots[db->htSize].offsetA; + } + } + } + //update header (close flags) + ptr = (Header_s*) db->mappedDb; + ptr->closeFailed = 0x00; //remove closeFailed flag + ptr->closeOk = 0x01; //set closeOk flag + msync(db->mappedDb, KISSDB_HEADER_SIZE, MS_SYNC); } - if( kdbShmemClose(db->shmem_ht_fd, db->shmem_ht_name) == Kdb_false) - return KISSDB_ERROR_CLOSE_SHM; - free(db->shmem_ht_name); - Kdb_unlock(&db->shmem_info->rwlock); - pthread_rwlock_destroy(&db->shmem_info->rwlock); - pthread_rwlock_destroy(&db->shmem_info->cache_rwlock); + //unmap whole database file + munmap(db->mappedDb, db->dbMappedSize); + db->mappedDb = NULL; - // free shared information - if (freeKdbShmemPtr(db->shmem_info, sizeof(Kdb_bool)) == Kdb_false) - return KISSDB_ERROR_UNMAP_SHM; - if (kdbShmemClose(db->shmem_info_fd, db->shmem_info_name) == Kdb_false) + //unmap shared hashtables + munmap(db->hashTables, db->htMappedSize); + db->hashTables = NULL; + //close shared memory for hashtables + if( kdbShmemClose(db->htFd, db->htName) == Kdb_false) + { + close(db->fd); + Kdb_unlock(&db->shared->rwlock); return KISSDB_ERROR_CLOSE_SHM; - free(db->shmem_info_name); + } + db->htFd = 0; -#ifdef __showTimeMeasurements - clock_gettime(CLOCK_ID, &mmapStart); -#endif + if(db->htName != NULL) + { + free(db->htName); + db->htName = NULL; + } - //update header (checksum and flags) - int mapFlag = PROT_WRITE | PROT_READ; - ptr = (Header_s*) mmap(NULL,KISSDB_HEADER_SIZE, mapFlag, MAP_SHARED, db->fd, 0); - if (ptr == MAP_FAILED) + //free rwlocks + Kdb_unlock(&db->shared->rwlock); + pthread_rwlock_destroy(&db->shared->rwlock); + + // unmap shared information + munmap(db->shared, sizeof(Shared_Data_s)); + db->shared = NULL; + + if (kdbShmemClose(db->sharedFd, db->sharedName) == Kdb_false) { close(db->fd); - return KISSDB_ERROR_IO; + return KISSDB_ERROR_CLOSE_SHM; } -#ifdef __checkerror - // generate checksum over database file (beginning at file offset [sizeof(ptr->KdbV) + sizeof(ptr->checksum)] up to EOF) - if( db->fd ) + db->sharedFd =0; + if(db->sharedName != NULL) { - crc = 0; - crc = (uint64_t) pcoCalcCrc32Csum(db->fd, sizeof(Header_s) ); - ptr->checksum = crc; - //printf("CLOSING ------ DB: %s, WITH CHECKSUM CALCULATED: %" PRIu64 " \n", db->shmem_ht_name, ptr->checksum); + free(db->sharedName); + db->sharedName = NULL; } -#endif - ptr->closeFailed = 0x00; //remove closeFailed flag - ptr->closeOk = 0x01; //set closeOk flag - - //sync changes with file - if( 0 != msync(ptr, KISSDB_HEADER_SIZE, MS_SYNC | MS_INVALIDATE)) + if(db->cacheName != NULL) { - close(db->fd); - return KISSDB_ERROR_IO; + free(db->cacheName); //free memory for name obtained by kdbGetShmName() function + db->cacheName = NULL; } - //unmap memory - if( 0 != munmap(ptr, KISSDB_HEADER_SIZE)) + if( db->fd) { close(db->fd); - return KISSDB_ERROR_IO; + db->fd = 0; } -#ifdef __showTimeMeasurements - clock_gettime(CLOCK_ID, &mmapEnd); - KdbDuration += getNsDuration(&mmapStart, &mmapEnd); - printf("mmap duration for => %f ms\n", (double)((double)KdbDuration/NANO2MIL)); -#endif - fsync(db->fd); - if( db->fd) - close(db->fd); + db->alreadyOpen = Kdb_false; + db->htSize = 0; + //db->cacheReferenced = 0; + db->keySize = 0; + db->valSize = 0; + db->htSizeBytes = 0; + db->htMappedSize = 0; + db->dbMappedSize = 0; + db->shmCreator = 0; + db->alreadyOpen = 0; + + //destroy named semaphore + if (-1 == sem_post(db->kdbSem)) //release semaphore + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(": sem_post() failed: "), + DLT_STRING(strerror(errno))); + } + if (-1 == sem_close(db->kdbSem)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(": sem_close() failed: "), + DLT_STRING(strerror(errno))); + } + if (-1 == sem_unlink(db->semName)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(": sem_unlink() failed: "), + DLT_STRING(strerror(errno))); + } + db->kdbSem = NULL; + if(db->semName != NULL) + { + free(db->semName); + db->semName = NULL; + } - db->already_open = Kdb_false; - //memset(db, 0, sizeof(KISSDB)); //todo check if necessary } else - //if caller is not the creator of the lock - Kdb_unlock(&db->shmem_info->rwlock); - return 0; -} - - -int KISSDB_get(KISSDB *db, const void *key, void *vbuf) -{ - Kdb_rdlock(&db->shmem_info->rwlock); + { + //if caller of close is not the last instance using the database + //unmap whole database file + munmap(db->mappedDb, db->dbMappedSize); + db->mappedDb = NULL; - uint8_t tmp[TMP_BUFFER_LENGTH]; - uint64_t current; - const uint8_t *kptr; - unsigned long klen, i; - long n = 0; - uint64_t checksum, backupChecksum, crc; - uint64_t hash = KISSDB_hash(key, db->key_size) % (uint64_t) db->hash_table_size; - int64_t offset, backupOffset, htoffset, checksumOffset, flagOffset; //lasthtoffset - Hashtable_slot_s *cur_hash_table; - -#ifdef __writeThrough - //if new one or more hashtables were appended, remap shared memory block to adress space - if (db->old_mapped_size < db->shmem_info->shmem_size) - { - Kdb_bool temp; - db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); - if(db->shmem_ht_fd < 0) - return KISSDB_ERROR_OPEN_SHM; - res = remapKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size, db->shmem_info->shmem_size); - if (res == Kdb_false) - return KISSDB_ERROR_REMAP_SHM; - db->old_mapped_size = db->shmem_info->shmem_size; - } -#endif + //unmap shared hashtables + munmap(db->hashTables, db->htMappedSize); + db->hashTables = NULL; - htoffset = KISSDB_HEADER_SIZE; //lasthtoffset - cur_hash_table = db->hash_tables;//pointer to current hashtable in memory - for (i = 0; i < db->shmem_info->num_hash_tables; ++i) - { - offset = cur_hash_table[hash].offsetA;//get fileoffset where the data can be found in the file -#ifdef __useBackups - //get information about current valid offset to latest written data - if(cur_hash_table[hash].current == 0x00) //valid is offsetA + if( db->fd) { - offset = cur_hash_table[hash].offsetA; - checksum = cur_hash_table[hash].checksumA; + close(db->fd); + db->fd = 0; } - else + if(db->htFd) { - offset = cur_hash_table[hash].offsetB; - checksum = cur_hash_table[hash].checksumB; + close(db->htFd); + db->htFd = 0; } -#endif - if (offset >= KISSDB_HEADER_SIZE) //if a valid offset is available in the slot - { - if (lseek(db->fd, offset, SEEK_SET) == -1) //move filepointer to this offset - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - kptr = (const uint8_t *) key; - klen = db->key_size; - while (klen) - { - n = (long) read(db->fd, tmp, (klen > sizeof(tmp)) ? sizeof(tmp) : klen); - if (n > 0) - { - if (memcmp(kptr, tmp, n))//if key does not match -> search in next hashtable - goto get_no_match_next_hash_table; - kptr += n; - klen -= (unsigned long) n; - } - else - { - Kdb_unlock(&db->shmem_info->rwlock); - return 1; /* not found */ - } - } - if (read(db->fd, vbuf, db->value_size) == db->value_size) //if key matches at the fileoffset -> read the value - { - //crc check for file content -#ifdef __useBackups - //only validate checksums at read if checksum of file is invalid - if (db->shmem_info->crc_invalid == Kdb_true) - { - //verify checksum of current key/value pair - crc = 0; - crc = (uint64_t) pcoCrc32(crc, (unsigned char*) vbuf, db->value_size); - if (checksum != crc) - { - //printf("KISSDB_get: WARNING: checksum invalid -> try to read from valid data block \n"); - //try to read valid data from backup - Hashtable_slot_s slot = cur_hash_table[hash]; - if (cur_hash_table[hash].current == 0x00) //current is offsetA, but Data there is corrupt--> so use offsetB as backupOffset - { - backupOffset = cur_hash_table[hash].offsetB; - backupChecksum = cur_hash_table[hash].checksumB; - checksumOffset = htoffset + (sizeof(Hashtable_slot_s) * hash + sizeof(slot.offsetA)); //offset that points to checksumA - current = 0x01; //current is offsetB - } - else - { - backupOffset = cur_hash_table[hash].offsetA; - backupChecksum = cur_hash_table[hash].checksumA; - checksumOffset = htoffset - + (sizeof(Hashtable_slot_s) * hash + sizeof(slot.offsetA) + sizeof(slot.checksumA) - + sizeof(slot.offsetB)); //offset that points to checksumB - current = 0x00; - } - flagOffset = htoffset - + (sizeof(Hashtable_slot_s) * hash + (sizeof(Hashtable_slot_s) - sizeof(slot.current))); //offset that points to currentflag + db->alreadyOpen = Kdb_false; + db->htSize = 0; + //db->cacheReferenced = 0; + db->keySize = 0; + db->valSize = 0; + db->htSizeBytes = 0; + db->htMappedSize = 0; + db->dbMappedSize = 0; + db->shmCreator = 0; + db->alreadyOpen = 0; - //seek to backup data - if (lseek(db->fd, backupOffset + db->key_size, SEEK_SET) == -1) //move filepointer to data of key-value pair //TODO make checksum over key AND data ?? - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } + Kdb_unlock(&db->shared->rwlock); - //verify checksum of backup key/value pair - //read from backup data - if (read(db->fd, vbuf, db->value_size) == db->value_size) //read value of backup Data block - { - //generate checksum of backup - crc = 0; - crc = (uint64_t) pcoCrc32(crc, (unsigned char*) vbuf, db->value_size); - if (crc == backupChecksum) //if checksum ok - { - //printf("KISSDB_get: WARNING: OVERWRITING CORRUPT DATA \n"); - //seek to corrupt data - if (lseek(db->fd, offset + db->key_size, SEEK_SET) == -1) //move filepointer to data of corrupt key-value pair - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - //overwrite corrupt data - if (write( db->fd, vbuf, db->value_size) != db->value_size ) //write value - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - //seek to header slot and update checksum of corrupt data (do not modify offsets) - if (lseek(db->fd, checksumOffset, SEEK_SET) == -1) //move to checksumX in file - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if (write( db->fd, &crc, sizeof(uint64_t)) != sizeof(uint64_t) ) //write checksumX to hashtbale slot - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - //update checksumX in memory - if (cur_hash_table[hash].current == 0x00) //current is offsetA, but Data there is corrupt--> so update checksumA with new checksum - cur_hash_table[hash].checksumA = crc; - else - cur_hash_table[hash].checksumB = crc; - //switch current valid to backup - - if (lseek(db->fd, flagOffset, SEEK_SET) == -1) //move to current flag in file - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if (write( db->fd, ¤t, sizeof(uint64_t)) != sizeof(uint64_t) ) //write current hashtable slot in file - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - //update current valid in memory - cur_hash_table[hash].current = current; - //fsync(db->fd) - Kdb_unlock(&db->shmem_info->rwlock); - return 0; /* success */ - } - else //if checksum not valid, return NOT FOUND - { - Kdb_unlock(&db->shmem_info->rwlock); - return 1; /* not found */ - } - } - else - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - } - } -#endif - Kdb_unlock(&db->shmem_info->rwlock); - return 0; /* success */ - } - else - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } + // unmap shared information + munmap(db->shared, sizeof(Shared_Data_s)); + db->shared = NULL; + + if(db->sharedFd) + { + close(db->sharedFd); + db->sharedFd = 0; } - else + if(db->htName != NULL) + { + free(db->htName); + db->htName = NULL; + } + if(db->sharedName != NULL) + { + free(db->sharedName); + db->sharedName = NULL; + } + if(db->cacheName != NULL) + { + free(db->cacheName); //free memory for name obtained by kdbGetShmName() function + db->cacheName = NULL; + } + //clean struct + if (-1 == sem_post(db->kdbSem)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(": sem_post() in close failed: "), + DLT_STRING(strerror(errno))); + } + if (-1 == sem_close(db->kdbSem)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(": sem_close() in close failed: "), + DLT_STRING(strerror(errno))); + } + db->kdbSem = NULL; + if(db->semName != NULL) { - Kdb_unlock(&db->shmem_info->rwlock); - return 1; /* not found */ + free(db->semName); + db->semName = NULL; } - //get_no_match_next_hash_table: cur_hash_table += db->hash_table_size + 1; - get_no_match_next_hash_table: //update lastht offset //lasthtoffset = htoffset - htoffset = cur_hash_table[db->hash_table_size].offsetA; // fileoffset to the next file-hashtable - cur_hash_table += (db->hash_table_size + 1); //pointer to the next memory-hashtable } - Kdb_unlock(&db->shmem_info->rwlock); - return 1; /* not found */ +#ifdef PFS_TEST + printf(" END: KISSDB_CLOSE \n"); +#endif + return 0; } -//TODO check current valid data to be deleted ? -int KISSDB_delete(KISSDB *db, const void *key) +int KISSDB_get(KISSDB* db, const void* key, void* vbuf, uint32_t bufsize, uint32_t* vsize) { - Kdb_wrlock(&db->shmem_info->rwlock); - - uint8_t tmp[TMP_BUFFER_LENGTH]; - //uint64_t current = 0x00; - const uint8_t *kptr; - long n; + const uint8_t* kptr; + DataBlock_s* block; + Hashtable_slot_s* hashTable; + int64_t offset; + Kdb_bool bCanContinue = Kdb_true; + Kdb_bool bKeyFound = Kdb_false; + uint64_t hash = 0; unsigned long klen, i; - //uint64_t crc = 0x00; - uint64_t hash = KISSDB_hash(key, db->key_size) % (uint64_t) db->hash_table_size; - //int64_t empty_offset = 0; - int64_t empty_offsetB = 0; - int64_t offset = 0; - int64_t htoffset = 0; - Hashtable_slot_s *cur_hash_table; -#ifdef __writeThrough - //if new hashtable was appended, remap shared memory block to adress space - if (db->old_mapped_size < db->shmem_info->shmem_size) + klen = strlen(key); + hash = KISSDB_hash(key, klen) % (uint64_t) db->htSize; + + if(db->htMappedSize < db->shared->htShmSize) { - Kdb_bool temp; - db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); - if(db->shmem_ht_fd < 0) - return KISSDB_ERROR_OPEN_SHM; - result = remapKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size, db->shmem_info->shmem_size); - if (result == Kdb_false) - return KISSDB_ERROR_REMAP_SHM; - db->old_mapped_size = db->shmem_info->shmem_size; + if ( Kdb_false == remapSharedHashtable(db->htFd, &db->hashTables, db->htMappedSize, db->shared->htShmSize)) + { + return KISSDB_ERROR_RESIZE_SHM; + } + else + { + db->htMappedSize = db->shared->htShmSize; + } + } + + hashTable = db->hashTables[0].slots; //pointer to first hashtable in memory at first slot + + if (db->dbMappedSize < db->shared->mappedDbSize) + { + db->mappedDb = mremap(db->mappedDb, db->dbMappedSize, db->shared->mappedDbSize, MREMAP_MAYMOVE); + if (db->mappedDb == MAP_FAILED) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":mremap error: !"), DLT_STRING(strerror(errno))); + return KISSDB_ERROR_IO; + } + else + { + db->dbMappedSize = db->shared->mappedDbSize; + } } -#endif - htoffset = KISSDB_HEADER_SIZE; - cur_hash_table = db->hash_tables; //pointer to current hashtable in memory - for (i = 0; i < db->shmem_info->num_hash_tables; ++i) + for (i = 0; i < db->shared->htNum; ++i) { - offset = cur_hash_table[hash].offsetA; //get fileoffset where the data can be found in the file - if (offset >= KISSDB_HEADER_SIZE) + //get information about current valid offset to latest written data + offset = (hashTable[hash].current == 0x00) ? hashTable[hash].offsetA : hashTable[hash].offsetB; // if 0x00 -> offsetA is latest else offsetB is latest + if(offset < 0) //deleted or invalidated data but search in next hashtable + { + bCanContinue = Kdb_false; //deleted datablock -> do not compare the key + } + else + { + bCanContinue = Kdb_true; //possible match -> compare the key + } + + if(Kdb_true == bCanContinue) { - if (lseek(db->fd, offset, SEEK_SET) == -1) + if( abs(offset) > db->dbMappedSize ) { - //set filepointer to Key value offset in file - Kdb_unlock(&db->shmem_info->rwlock); return KISSDB_ERROR_IO; } - kptr = (const uint8_t *) key; - klen = db->key_size; - while (klen) + if (offset >= KISSDB_HEADER_SIZE) //if a valid offset is available in the slot { - n = (long) read(db->fd, tmp, (klen > sizeof(tmp)) ? sizeof(tmp) : klen); - if (n > 0) + block = (DataBlock_s*) (db->mappedDb + offset); + kptr = (const uint8_t*) key; + if (klen > 0) { - if (memcmp(kptr, tmp, n))//if key does not match, search in next hashtable - goto get_no_match_next_hash_table; - kptr += n; - klen -= (unsigned long) n; + if (memcmp(kptr, block->key, klen) + || strlen(block->key) != klen) //if search key does not match with key in file + { + bKeyFound = Kdb_false; + } + else + { + bKeyFound = Kdb_true; + } } - else + if(Kdb_true == bKeyFound) { - Kdb_unlock(&db->shmem_info->rwlock); - return 1; /* not found */ + //copy found value if buffer is big enough + if(bufsize >= block->valSize) + { + memcpy(vbuf, block->value, block->valSize); + } + *(vsize) = block->valSize; + return 0; /* success */ } } - //TODO: mmap Hashtable slot structure to avoid seeking -> align hashtables at a multiple of a pagesize -#ifdef __useFileMapping - empty_offsetB = -(offset + (db->key_size + db->value_size)); //todo check if offset is rewritten in put function ! - cur_hash_table[hash].offsetB = empty_offsetB; - cur_hash_table[hash].checksumA = 0x00; - cur_hash_table[hash].checksumB = 0x00; - cur_hash_table[hash].current = 0x00; - int testoffset= lseek(fd, 0, SEEK_CUR); //filepointer position - int myoffset = htoffset + (sizeof(Hashtable_slot_s) * hash); - - printf("Endoffset in file: %d , Offset for mmap: %d , size for mmap: %d \n", testoffset, myoffset, sizeof(Hashtable_slot_s)); - - //mmap the current hashtable slot - int mapFlag = PROT_WRITE | PROT_READ; - printf("In Delete: filedes: %d\n", db->fd); - htSlot = (Hashtable_slot_s*) mmap(NULL, sizeof(Hashtable_slot_s), mapFlag, MAP_SHARED, db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash) ); //TODO offset must be a multiple of pagesize - if (htSlot == MAP_FAILED) - { - printf("MMAP ERROR !\n"); - close(db->fd); - return KISSDB_ERROR_IO; - } - //do changes to slot in file - htSlot->offsetA = empty_offset; - htSlot->checksumA = 0x00; - htSlot->offsetB = empty_offsetB; - htSlot->checksumB = 0x00; - htSlot->current = 0x00; - - //sync changes with file - if (0 != msync(htSlot, sizeof(Hashtable_slot_s), MS_SYNC | MS_INVALIDATE)) - { - close(db->fd); - return KISSDB_ERROR_IO; - } - //unmap memory - if (0 != munmap(htSlot, sizeof(Hashtable_slot_s))) - { - close(db->fd); - return KISSDB_ERROR_IO; - } -#endif - if (lseek(db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash), SEEK_SET) == -1) //move Filepointer to used slot in file-hashtable. + else { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + return 1; /* not found */ } + } + hashTable = (Hashtable_slot_s*) ((char*) hashTable + sizeof(Hashtable_s)); //pointer to the next memory-hashtable + } + return 1; /* not found */ +} -#ifndef __useBackups - cur_hash_table[hash].offsetA = -offset; //negate offset in hashtable that points to the data - empty_offset = -offset; - //update hashtable slot in file header (delete existing offset information) - if (write( db->fd, &empty_offset, sizeof(int64_t)) != sizeof(int64_t) ) //mark slot in file-hashtable as deleted - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } -#endif -#ifdef __useBackups - //negate offsetB, delete checksums and current flag in memory - cur_hash_table[hash].offsetA = -offset; //negate offset in hashtable that points to the data - empty_offsetB = -(offset + (db->key_size + db->value_size)); - cur_hash_table[hash].checksumA = 0x00; - cur_hash_table[hash].offsetB = empty_offsetB; - cur_hash_table[hash].checksumB = 0x00; - cur_hash_table[hash].current = 0x00; - if (write( db->fd, &cur_hash_table[hash], sizeof(Hashtable_slot_s)) != sizeof(Hashtable_slot_s) ) //write updated data in the file-hashtable slot - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } -#endif - //TODO currently, no synchronus Filedescriptor is used!!!! fsync after fflush is needed to do synchronus writes - //fsync(db->fd) // associating a file stream with a synchronous file descriptor means that an fsync() call is not needed on the file descriptor after the fflush() - Kdb_unlock(&db->shmem_info->rwlock); - return 0; /* success */ +int KISSDB_delete(KISSDB* db, const void* key, int32_t* bytesDeleted) +{ + const uint8_t* kptr; + DataBlock_s* backupBlock; + DataBlock_s* block; + Hashtable_slot_s* hashTable; + int64_t backupOffset = 0; + int64_t offset = 0; + Kdb_bool bCanContinue = Kdb_true; + Kdb_bool bKeyFound = Kdb_false; + uint64_t hash = 0; + uint64_t crc = 0x00; + unsigned long klen, i; + + klen = strlen(key); + hash = KISSDB_hash(key, klen) % (uint64_t) db->htSize; + *(bytesDeleted) = PERS_COM_ERR_NOT_FOUND; + + if(db->htMappedSize < db->shared->htShmSize) + { + if ( Kdb_false == remapSharedHashtable(db->htFd, &db->hashTables, db->htMappedSize, db->shared->htShmSize)) + { + return KISSDB_ERROR_RESIZE_SHM; } else { - Kdb_unlock(&db->shmem_info->rwlock); - return 1; /* not found */ //if no offset is found at hashed position in ht + db->htMappedSize = db->shared->htShmSize; } - get_no_match_next_hash_table: htoffset = cur_hash_table[db->hash_table_size].offsetA; // fileoffset to next ht in file - cur_hash_table += (db->hash_table_size + 1); //pointer to next hashtable in memory } - Kdb_unlock(&db->shmem_info->rwlock); - return 1; /* not found */ -} - -int KISSDB_put(KISSDB *db, const void *key, const void *value) -{ - Kdb_wrlock(&db->shmem_info->rwlock); - uint8_t tmp[TMP_BUFFER_LENGTH]; - uint64_t current = 0x00; - const uint8_t *kptr; - unsigned long klen, i; - uint64_t hash = KISSDB_hash(key, db->key_size) % (uint64_t) db->hash_table_size; - int64_t offset, endoffset, htoffset, lasthtoffset; - Hashtable_slot_s *cur_hash_table; - Kdb_bool result = Kdb_false; - Kdb_bool temp = Kdb_false; - uint64_t crc = 0x00; - long n; - char delimiter[8] = "||||||||"; + hashTable = db->hashTables->slots; //pointer to current hashtable in memory -#ifdef __writeThrough - //if new hashtable was appended, remap shared memory block to adress space - if(db->old_mapped_size < db->shmem_info->shmem_size) + //remap database file if in the meanwhile another process added new data (key value pairs / hashtables) to the file + if (db->dbMappedSize < db->shared->mappedDbSize) { - db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); - if(db->shmem_ht_fd < 0) - return KISSDB_ERROR_OPEN_SHM; - res = remapKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size,db->shmem_info->shmem_size); - if (res == Kdb_false) - return KISSDB_ERROR_REMAP_SHM; - db->old_mapped_size = db->shmem_info->shmem_size; + db->mappedDb = mremap(db->mappedDb, db->dbMappedSize, db->shared->mappedDbSize, MREMAP_MAYMOVE); + if (db->mappedDb == MAP_FAILED) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":mremap error: !"), DLT_STRING(strerror(errno))); + return KISSDB_ERROR_IO; + } + db->dbMappedSize = db->shared->mappedDbSize; } -#endif - lasthtoffset = htoffset = KISSDB_HEADER_SIZE; - cur_hash_table = db->hash_tables; //pointer to current hashtable in memory - for (i = 0; i < db->shmem_info->num_hash_tables; ++i) + for (i = 0; i < db->shared->htNum; ++i) { - offset = cur_hash_table[hash].offsetA; //fileoffset to data in file - if (offset >= KISSDB_HEADER_SIZE || offset < 0) //if a key with same hash is already in this slot or the same key must be overwritten + //get information about current valid offset to latest written data + if (hashTable[hash].current == 0x00) //valid is offsetA { - // if slot is marked as deleted, use this slot and negate the offset in order to reuse the existing data block - if(offset < 0) + offset = hashTable[hash].offsetA; + backupOffset = hashTable[hash].offsetB; + } + else + { + offset = hashTable[hash].offsetB; + backupOffset = hashTable[hash].offsetA; + } + if (offset < 0) //deleted or invalidated data but search in next hashtable + { + bCanContinue = Kdb_false; + } + else + { + bCanContinue = Kdb_true; + } + if (Kdb_true == bCanContinue) + { + if (offset >= KISSDB_HEADER_SIZE) { - offset = -offset; //get original offset where data was deleted - //printf("Overwriting slot for key: [%s] which was deleted before, offsetA: %d \n",key, offset); - if (lseek(db->fd, offset, SEEK_SET) == -1) //move filepointer to fileoffset where the key can be found - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if (write( db->fd, key, db->key_size) != db->key_size ) //write key - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if (write( db->fd, value, db->value_size) != db->value_size ) //write value + if( abs(offset) > db->dbMappedSize || abs(backupOffset) > db->dbMappedSize) { - Kdb_unlock(&db->shmem_info->rwlock); return KISSDB_ERROR_IO; } - // write same key and value again here because slot was deleted an can be reused like an initial write -#ifdef __useBackups - if (write( db->fd, key, db->key_size) != db->key_size ) //write key + block = (DataBlock_s*) (db->mappedDb + offset); + kptr = (const uint8_t*) key; //pointer to search key + if (klen > 0) { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if (write( db->fd, value, db->value_size) != db->value_size ) //write value - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } -#endif - //seek back to hashtbale slot - if (lseek(db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash), SEEK_SET) == -1) //move to beginning of hashtable slot in file - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - -#ifndef __useBackups - cur_hash_table[hash].offsetA = offset; //write the offset to the data in the memory-hashtable slot - if (write( db->fd, &offset, sizeof(int64_t)) != sizeof(int64_t) ) //write the offsetA to the data in the file-hashtable slot - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + if (memcmp(kptr, block->key, klen) || strlen(block->key) != klen) //if search key does not match with key in file + { + bKeyFound = Kdb_false; + } + else + { + bKeyFound = Kdb_true; + } } -#endif - -#ifdef __useBackups - crc = 0x00; - crc = (uint32_t) pcoCrc32(crc, (unsigned char*)value, db->value_size); - cur_hash_table[hash].offsetA = offset; //write the offset to the data in the memory-hashtable slot - cur_hash_table[hash].checksumA = crc; - offset += (db->key_size + db->value_size); - cur_hash_table[hash].offsetB = offset; //write the offset to the data in the memory-hashtable slot - cur_hash_table[hash].checksumB = crc; - cur_hash_table[hash].current = 0x00; - - if (write( db->fd, &cur_hash_table[hash], sizeof(Hashtable_slot_s)) != sizeof(Hashtable_slot_s) ) //write updated data in the file-hashtable slot + /* data to be deleted was found + write "deleted block delimiters" for both blocks and delete key / value */ + if (Kdb_true == bKeyFound) { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + block->delimStart = (offset < backupOffset) ? DATA_BLOCK_A_DELETED_START_DELIMITER : DATA_BLOCK_B_DELETED_START_DELIMITER; + //memset(block->key, 0, db->keySize); //do not delete key -> used in hashtable rebuild + memset(block->value, 0, db->valSize); + block->valSize = 0; + crc = 0x00; + crc = (uint32_t) pcoCrc32(crc, (unsigned char*)block->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t) ); + block->crc = crc; + block->delimEnd = (offset < backupOffset) ? DATA_BLOCK_A_DELETED_END_DELIMITER : DATA_BLOCK_B_DELETED_END_DELIMITER; + + backupBlock = (DataBlock_s*) (db->mappedDb + backupOffset); //map data and backup block + + backupBlock->delimStart = (backupOffset < offset) ? DATA_BLOCK_A_DELETED_START_DELIMITER : DATA_BLOCK_B_DELETED_START_DELIMITER; + //memset(backupBlock->key, 0, db->keySize); + memset(backupBlock->value, 0, db->valSize); + backupBlock->valSize = 0; + crc = 0x00; + crc = (uint32_t) pcoCrc32(crc, (unsigned char*)backupBlock->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t) ); + backupBlock->crc = crc; + backupBlock->delimEnd = (backupOffset < offset) ? DATA_BLOCK_A_DELETED_END_DELIMITER : DATA_BLOCK_B_DELETED_END_DELIMITER; + + + //negate offsetB, delete checksums and current flag in memory + hashTable[hash].offsetA = -hashTable[hash].offsetA; //negate offset in hashtable that points to the data + hashTable[hash].offsetB = -hashTable[hash].offsetB; + hashTable[hash].current = 0x00; + + *(bytesDeleted) = block->valSize; + return 0; /* success */ } -#endif - //fsync(db->fd) //associating a file stream with a synchronous file descriptor means that an fsync() call is not needed on the file descriptor after the fflush() - Kdb_unlock(&db->shmem_info->rwlock); - return 0; /* success */ } - - //overwrite existing if key matches - // if cur_hash_table[hash].current == 0x00 -> offsetA is latest so write to offsetB else offsetB is latest and write to offsetA -#ifdef __useBackups - if( cur_hash_table[hash].current == 0x00 ) - offset = cur_hash_table[hash].offsetA; //0x00 -> offsetA is latest else - offset = cur_hash_table[hash].offsetB; //else offsetB is latest -#endif - if (lseek(db->fd, offset, SEEK_SET) == -1) //move filepointer to fileoffset where valid data can be found { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + return 1; /* not found */ //if no offset is found at hashed position in slots } + } + hashTable = (Hashtable_slot_s*) ((char*) hashTable + sizeof(Hashtable_s)); //pointer to the next memory-hashtable + } + return 1; /* not found */ +} - kptr = (const uint8_t *) key; //pointer to search key - klen = db->key_size; - while (klen) - { - n = (long) read(db->fd, tmp, (klen > sizeof(tmp)) ? sizeof(tmp) : klen); - if (n > 0) - { - if (memcmp(kptr, tmp, n)) //if search key does not match with key in file - goto put_no_match_next_hash_table; - kptr += n; - klen -= (unsigned long) n; - } - } +// To improve write amplifiction: sort the keys at writeback for sequential write +//return offset where key would be written if Kissdb_put with same key is called +//int determineKeyOffset(KISSDB* db, const void* key) +//{ +// /* +// * - hash the key, +// * - go through hashtables and get corresponding offset to hash +// * - if offset is negative return the inverse offset +// * - if key matches at file offset return this offset +// */ +// return 0; +//} - //if key matches -> seek to currently non valid data block for this key -#ifdef __useBackups - if( cur_hash_table[hash].current == 0x00 ) - offset = cur_hash_table[hash].offsetB; // 0x00 -> offsetA is latest so write new data to offsetB which holds old data - else - offset = cur_hash_table[hash].offsetA; // offsetB is latest so write new data to offsetA which holds old data - if (lseek(db->fd, offset, SEEK_SET) == -1)//move filepointer to fileoffset where backup data can be found - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if (write( db->fd, key, db->key_size) != db->key_size ) //write key - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } -#endif - if (write( db->fd, value, db->value_size) != db->value_size ) - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - // seek back to slot in header for update of checksum and flag - if (lseek(db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash), SEEK_SET) == -1) //move to beginning of hashtable slot in file + + +int KISSDB_put(KISSDB* db, const void* key, const void* value, int valueSize, int32_t* bytesWritten) +{ + const uint8_t* kptr; + DataBlock_s* backupBlock; + DataBlock_s* block; + Hashtable_s* hashtable; + Hashtable_s* htptr; + Hashtable_slot_s* hashTable; + int64_t offset, backupOffset, endoffset; + Kdb_bool bKeyFound = Kdb_false; + Kdb_bool result = Kdb_false; + Kdb_bool temp = Kdb_false; + uint64_t crc = 0x00; + uint64_t hash = 0; + unsigned long klen, i; + + klen = strlen(key); + hash = KISSDB_hash(key, klen) % (uint64_t) db->htSize; + *(bytesWritten) = 0; + + if(db->htMappedSize < db->shared->htShmSize) + { + if ( Kdb_false == remapSharedHashtable(db->htFd, &db->hashTables, db->htMappedSize, db->shared->htShmSize)) + { + return KISSDB_ERROR_RESIZE_SHM; + } + else + { + db->htMappedSize = db->shared->htShmSize; + } + } + + hashTable = db->hashTables->slots; //pointer to current hashtable in memory + + //remap database file (only necessary here in writethrough mode) if in the meanwhile another process added new data (key value pairs / hashtables) to the file + if (db->dbMappedSize < db->shared->mappedDbSize) + { + db->mappedDb = mremap(db->mappedDb, db->dbMappedSize, db->shared->mappedDbSize, MREMAP_MAYMOVE); + if (db->mappedDb == MAP_FAILED) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":mremap error: !"), DLT_STRING(strerror(errno))); + return KISSDB_ERROR_IO; + } + db->dbMappedSize = db->shared->mappedDbSize; + } + + + for (i = 0; i < db->shared->htNum; ++i) + { + offset = hashTable[hash].offsetA; //fileoffset to data in file + if (offset >= KISSDB_HEADER_SIZE || offset < 0) //if a key with same hash is already in this slot or the same key must be overwritten + { + if( abs(offset) > db->dbMappedSize ) { - Kdb_unlock(&db->shmem_info->rwlock); return KISSDB_ERROR_IO; } - //generate crc for value - crc = 0x00; - crc = (uint64_t) pcoCrc32(crc, (unsigned char*)value, db->value_size); - current = 0x00; - Hashtable_slot_s slot = cur_hash_table[hash]; + // if slot is marked as deleted, use this slot and negate the offset in order to reuse the existing data block + if(offset < 0) + { + offset = -offset; //get original offset where data was deleted + //printf("Overwriting slot for key: [%s] which was deleted before, offsetA: %d \n",key, offset); + writeDualDataBlock(db, offset, i, key, klen, value, valueSize); + hashTable[hash].offsetA = offset; //write the offset to the data in the memory-hashtable slot + offset += sizeof(DataBlock_s); + hashTable[hash].offsetB = offset; //write the offset to the second databloxk in the memory-hashtable slot + hashTable[hash].current = 0x00; + *(bytesWritten) = valueSize; - // check current flag and decide what parts of hashtable slot in file must be updated - if( cur_hash_table[hash].current == 0x00 ) //offsetA is latest -> modify settings of B + return 0; /* success */ + } + //overwrite existing if key matches + offset = (hashTable[hash].current == 0x00) ? hashTable[hash].offsetA : hashTable[hash].offsetB; // if 0x00 -> offsetA is latest else offsetB is latest + block = (DataBlock_s*) (db->mappedDb + offset); + kptr = (const uint8_t*) key; //pointer to search key + if (klen > 0) { - int seek = sizeof(slot.offsetA) + sizeof(slot.checksumA) + sizeof(slot.offsetB); - lseek(db->fd, seek , SEEK_CUR); //move to checksumB in file - if( write( db->fd, &crc, sizeof(uint64_t)) != sizeof(uint64_t)) //write checksumB to file + if (memcmp(kptr, block->key, klen) + || strlen(block->key) != klen) //if search key does not match with key in file { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + //if key does not match -> search in next hashtable + bKeyFound = Kdb_false; } - current = 0x01; - if( write( db->fd, ¤t, sizeof(uint64_t)) != sizeof(uint64_t)) //write current to hashtbale slot + else { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + bKeyFound = Kdb_true; } - cur_hash_table[hash].checksumB = crc; - cur_hash_table[hash].current = current; } - else //offsetB is latest -> modify settings of A + if(Kdb_true == bKeyFound) { + backupOffset = (hashTable[hash].current == 0x00) ? hashTable[hash].offsetB : hashTable[hash].offsetA; // if 0x00 -> offsetB is latest backup else offsetA is latest - int seek = sizeof(slot.offsetA); - lseek(db->fd, seek , SEEK_CUR); //move to checksumA in file - if( write( db->fd, &crc, sizeof(uint64_t)) != sizeof(uint64_t)) //write checksumA to file - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - seek = sizeof(slot.offsetB) + sizeof(slot.checksumB);; - lseek(db->fd, seek , SEEK_CUR); //move to checksumA in file - current = 0x00; - if( write( db->fd, ¤t, sizeof(uint64_t)) != sizeof(uint64_t))//write current to hashtbale slot - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - cur_hash_table[hash].checksumA = crc; - cur_hash_table[hash].current = current; + //ALSO OVERWRITE LATEST VALID BLOCK to improve write amplification factor + block->delimStart = (offset < backupOffset) ? DATA_BLOCK_A_START_DELIMITER : DATA_BLOCK_B_START_DELIMITER; + block->valSize = valueSize; + memcpy(block->value,value, block->valSize); + block->htNum = i; + crc = 0x00; + crc = (uint32_t) pcoCrc32(crc, (unsigned char*)block->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t) ); + block->crc = crc; + block->delimEnd = (offset < backupOffset) ? DATA_BLOCK_A_END_DELIMITER : DATA_BLOCK_B_END_DELIMITER; + + //if key matches -> seek to currently non valid data block for this key + backupOffset = (hashTable[hash].current == 0x00) ? hashTable[hash].offsetB : hashTable[hash].offsetA; // if 0x00 -> offsetB is latest backup else offsetA is latest + backupBlock = (DataBlock_s*) (db->mappedDb + backupOffset); + //backupBlock->delimStart = DATA_BLOCK_START_DELIMITER; + backupBlock->delimStart = (backupOffset < offset) ? DATA_BLOCK_A_START_DELIMITER : DATA_BLOCK_B_START_DELIMITER; + backupBlock->valSize = valueSize; + memcpy(backupBlock->value,value, backupBlock->valSize); + backupBlock->htNum = i; + crc = 0x00; + crc = (uint32_t) pcoCrc32(crc, (unsigned char*)backupBlock->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t) ); + backupBlock->crc = crc; + //backupBlock->delimEnd = DATA_BLOCK_END_DELIMITER; + backupBlock->delimEnd = (backupOffset < offset) ? DATA_BLOCK_A_END_DELIMITER : DATA_BLOCK_B_END_DELIMITER; + // check current flag and decide what parts of hashtable slot in file must be updated + hashTable[hash].current = (hashTable[hash].current == 0x00) ? 0x01 : 0x00; // if 0x00 -> offsetA is latest -> set to 0x01 else /offsetB is latest -> modify settings of A set 0x00 + *(bytesWritten) = valueSize; + + return 0; //success } - Kdb_unlock(&db->shmem_info->rwlock); - return 0; //success } else //if key is not already inserted { /* add new data if an empty hash table slot is discovered */ - if (lseek(db->fd, 0, SEEK_END) == -1) //filepointer to the end of the file + endoffset = db->shared->mappedDbSize; + if ( -1 == endoffset) //filepointer to the end of the file { - Kdb_unlock(&db->shmem_info->rwlock); return KISSDB_ERROR_IO; } - endoffset = lseek(db->fd, 0, SEEK_CUR); //filepointer position - if (write( db->fd, key, db->key_size) != db->key_size ) + + //truncate file -> data + backup block + if( ftruncate(db->fd, endoffset + ( sizeof(DataBlock_s) * 2) ) < 0) { - Kdb_unlock(&db->shmem_info->rwlock); return KISSDB_ERROR_IO; } - if (write( db->fd, value, db->value_size) != db->value_size ) + + db->mappedDb = mremap(db->mappedDb, db->dbMappedSize, db->shared->mappedDbSize + (sizeof(DataBlock_s) * 2), MREMAP_MAYMOVE); + if (db->mappedDb == MAP_FAILED) { - Kdb_unlock(&db->shmem_info->rwlock); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":mremap error: !"),DLT_STRING(strerror(errno))); return KISSDB_ERROR_IO; } - // write same key and value again here --> initial write -#ifdef __useBackups - if (write( db->fd, key, db->key_size) != db->key_size ) - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if (write( db->fd, value, db->value_size) != db->value_size ) - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } -#endif - if (lseek(db->fd, htoffset + (sizeof(Hashtable_slot_s) * hash), SEEK_SET) == -1) //move filepointer to file-hashtable slot in file (offsetA) - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } -#ifndef __useBackups - if (write( db->fd, &endoffset, sizeof(int64_t)) != sizeof(int64_t) ) //write the offsetA to the data in the file-hashtable slot - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - cur_hash_table[hash].offsetA = endoffset; //write the offsetA to the data in the memory-hashtable slot -#endif + db->shared->mappedDbSize = db->shared->mappedDbSize + (sizeof(DataBlock_s) * 2); //shared info about database file size + db->dbMappedSize = db->shared->mappedDbSize; //local info about mapped size of file -#ifdef __useBackups - crc = 0x00; - crc = (uint64_t) pcoCrc32(crc, (unsigned char*) value, db->value_size); - offset = endoffset + (db->key_size + db->value_size); - cur_hash_table[hash].offsetA = endoffset; //write the offsetA to the data in the memory-hashtable slot - cur_hash_table[hash].checksumA = crc; - cur_hash_table[hash].offsetB = offset; //write the offset to the data in the memory-hashtable slot - cur_hash_table[hash].checksumB = crc; - cur_hash_table[hash].current = 0x00; - current = 0x00; //current - - if (write( db->fd, &cur_hash_table[hash], sizeof(Hashtable_slot_s)) != sizeof(Hashtable_slot_s) ) //write updated data in the file-hashtable slot - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } -#endif - //fsync(db->fd) // associating a file stream with a synchronous file descriptor means that an fsync() call is not needed on the file descriptor after the fflush() - Kdb_unlock(&db->shmem_info->rwlock); - return 0; /* success */ - } - put_no_match_next_hash_table: lasthtoffset = htoffset; - htoffset = cur_hash_table[db->hash_table_size].offsetA; // fileoffset to the next file-hashtable - cur_hash_table += (db->hash_table_size + 1); //pointer to the next memory-hashtable - } + writeDualDataBlock(db, endoffset, i, key, klen, value, valueSize); - /* if no existing slots, add a new page of hash table entries */ - if (lseek(db->fd, 0, SEEK_END) == -1) //Filepointer to the end of file - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if(db->shmem_info->num_hash_tables > 0) //only write delimiter if first hashtable has been written (first delimiter is written by open call) - { - if (write( db->fd, &delimiter, sizeof(delimiter)) != sizeof(delimiter) ) //write delimiter - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + //update hashtable entry + offset = endoffset + sizeof(DataBlock_s); + hashTable[hash].offsetA = endoffset; //write the offsetA to the data in the memory-hashtable slot + hashTable[hash].offsetB = offset; //write the offset to the data in the memory-hashtable slot + hashTable[hash].current = 0x00; + + *(bytesWritten) = valueSize; + return 0; /* success */ } + hashTable = (Hashtable_slot_s*) ((char*) hashTable + sizeof(Hashtable_s)); //pointer to the next memory-hashtable } - endoffset = lseek(db->fd, 0, SEEK_CUR); - //if new size would exceed old shared memory size-> allocate additional memory to shared memory (+ db->hash_table_size_bytes) - if( (db->hash_table_size_bytes * (db->shmem_info->num_hash_tables + 1)) > db->shmem_info->shmem_size) + //if new size would exceed old shared memory size for hashtables-> allocate additional memory to shared memory (+ db->htSizeBytes) + if( (db->htSizeBytes * (db->shared->htNum + 1)) > db->shared->htShmSize) { - if (db->shmem_ht_fd <= 0) + //munlockall(); + if (db->htFd <= 0) { - db->shmem_ht_fd = kdbShmemOpen(db->shmem_ht_name, db->old_mapped_size, &temp); - if(db->shmem_ht_fd < 0) + db->htFd = kdbShmemOpen(db->htName, db->htMappedSize, &temp); + if(db->htFd < 0) + { return KISSDB_ERROR_OPEN_SHM; + } } - result = resizeKdbShmem(db->shmem_ht_fd, &db->hash_tables, db->old_mapped_size, db->old_mapped_size + db->hash_table_size_bytes); + result = resizeKdbShmem(db->htFd, &db->hashTables, db->htMappedSize, db->htMappedSize + db->htSizeBytes); if (result == Kdb_false) { return KISSDB_ERROR_RESIZE_SHM; } else { - db->shmem_info->shmem_size = db->old_mapped_size + db->hash_table_size_bytes; - db->old_mapped_size = db->shmem_info->shmem_size; + db->shared->htShmSize = db->htMappedSize + db->htSizeBytes; + db->htMappedSize = db->shared->htShmSize; } + //mlockall(MCL_FUTURE); } - //if( currentHtOffset <= db->old_mapped_size / sizeof(Hashtable_slot_s) ) - cur_hash_table = &(db->hash_tables[(db->hash_table_size + 1) * db->shmem_info->num_hash_tables]); - //else - // return KISSDB_ERROR_ACCESS_VIOLATION; - memset(cur_hash_table, 0, db->hash_table_size_bytes); //hashtable init - cur_hash_table[hash].offsetA = endoffset + db->hash_table_size_bytes; /* where new entry will go (behind the new Ht that gets written)*/ - -#ifdef __useBackups - crc = 0x00; - crc = (uint64_t) pcoCrc32(crc, (unsigned char*)value, db->value_size); - cur_hash_table[hash].checksumA = crc; - cur_hash_table[hash].checksumB = crc; - cur_hash_table[hash].offsetB = cur_hash_table[hash].offsetA + (db->key_size + db->value_size);//write the offset to the data in the memory-hashtable slot - cur_hash_table[hash].current = 0x00; -#endif - - // write new hashtable at the end of the file - if (write( db->fd, cur_hash_table, db->hash_table_size_bytes) != db->hash_table_size_bytes ) - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - // write key behind new hashtable - if (write( db->fd, key, db->key_size) != db->key_size ) + /* if no existing slots, add a new page of hash table entries */ + endoffset = db->shared->mappedDbSize; + if ( -1 == endoffset) //filepointer to the end of the file { - Kdb_unlock(&db->shmem_info->rwlock); return KISSDB_ERROR_IO; } - // write value behind key - if (write( db->fd, value, db->value_size) != db->value_size ) + //truncate file in order to save new hashtable + two Datablocks (this does not modify filedescriptor) + if (ftruncate(db->fd, endoffset + db->htSizeBytes + (sizeof(DataBlock_s) * 2)) < 0) { - Kdb_unlock(&db->shmem_info->rwlock); return KISSDB_ERROR_IO; } - // write same key and value again here --> initial write -#ifdef __useBackups - if (write( db->fd, key, db->key_size) != db->key_size ) + + db->mappedDb = mremap(db->mappedDb, db->dbMappedSize, db->shared->mappedDbSize + (db->htSizeBytes + (sizeof(DataBlock_s) * 2)), MREMAP_MAYMOVE); + if (db->mappedDb == MAP_FAILED) { - Kdb_unlock(&db->shmem_info->rwlock); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":mremap error: !"),DLT_STRING(strerror(errno))); return KISSDB_ERROR_IO; } - if (write( db->fd, value, db->value_size) != db->value_size ) + db->shared->mappedDbSize = db->shared->mappedDbSize + (db->htSizeBytes + (sizeof(DataBlock_s) * 2)); + db->dbMappedSize = db->shared->mappedDbSize; + + //prepare new hashtable in shared memory + hashtable = &(db->hashTables[db->shared->htNum]); + memset(hashtable, 0, db->htSizeBytes); //hashtable init + hashtable->delimStart = HASHTABLE_START_DELIMITER; + hashtable->delimEnd = HASHTABLE_END_DELIMITER; + hashtable->crc = 0x00; + hashTable = hashtable->slots; //pointer to the next memory-hashtable + hashTable[hash].offsetA = endoffset + db->htSizeBytes; /* where new entry will go (behind the new Ht that gets written)*/ + hashTable[hash].offsetB = hashTable[hash].offsetA + sizeof(DataBlock_s);//write the offset to the data in the memory-hashtable slot + hashTable[hash].current = 0x00; + + htptr = (Hashtable_s*) (db->mappedDb + endoffset); + //copy hashtable in shared memory to mapped hashtable in file + memcpy(htptr, hashtable, db->htSizeBytes); + //write data behind new hashtable + writeDualDataBlock(db, endoffset + db->htSizeBytes, db->shared->htNum, key, klen, value, valueSize); + //if a hashtable exists, update link to new hashtable in previous hashtable + if (db->shared->htNum) { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; + db->hashTables[db->shared->htNum -1].slots[HASHTABLE_SLOT_COUNT].offsetA = endoffset; //update link to new hashtable in previous hashtable } -#endif + ++db->shared->htNum; + + *(bytesWritten) = valueSize; - //if a hashtable exists, update link to new hashtable - if (db->shmem_info->num_hash_tables) - { - if (lseek(db->fd, lasthtoffset + (sizeof(Hashtable_slot_s) * db->hash_table_size), SEEK_SET) == -1) - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - if (write( db->fd, &endoffset, sizeof(int64_t)) != sizeof(int64_t) ) - { - Kdb_unlock(&db->shmem_info->rwlock); - return KISSDB_ERROR_IO; - } - db->hash_tables[((db->hash_table_size + 1) * (db->shmem_info->num_hash_tables - 1)) + db->hash_table_size].offsetA = endoffset; //update link to new hashtable in old hashtable - } - ++db->shmem_info->num_hash_tables; - //fsync(db->fd) - Kdb_unlock(&db->shmem_info->rwlock); return 0; /* success */ } @@ -1242,34 +1280,25 @@ int KISSDB_put(KISSDB *db, const void *key, const void *value) /* * prints the offsets stored in the shared Hashtable */ -void printSharedHashtable(KISSDB *db) +void printSharedHashtable(KISSDB* db) { - Hashtable_slot_s *cur_hash_table; - cur_hash_table = db->hash_tables; - unsigned long k; - unsigned long x = (db->hash_table_size * db->shmem_info->num_hash_tables); - //printf("Address of SHARED HT_NUMBER: %p \n", &db->shmem_info->num_hash_tables); - printf("Address of SHARED HEADER: %p \n", &cur_hash_table); - Header_s* ptr; - printf("HT Struct sizes: %d, %d, %d, %d,%d, %d, %d, %d\n", sizeof(ptr->KdbV), sizeof(ptr->checksum), sizeof(ptr->closeFailed), sizeof(ptr->closeOk), sizeof(ptr->hash_table_size),sizeof(ptr->key_size),sizeof(ptr->value_size),sizeof(ptr->delimiter)); - printf("HEADER SIZE: %d \n", sizeof(Header_s)); - printf("Hashtable_slot_s SIZE: %d \n", sizeof(Hashtable_slot_s)); - for (k = 0; k < x; k++) + unsigned long k=0; + int i = 0; + + for(i =0 ; i< db->shared->htNum; i++) { - if (db->hash_tables[k].offsetA != 0) + for (k = 0; k <= db->htSize; k++) { - printf("offsetA [%lu]: %" PRId64 " \n", k, db->hash_tables[k].offsetA); - printf("checksumA[%lu]: %" PRIu64 " \n", k, db->hash_tables[k].checksumA); - printf("offsetB [%lu]: %" PRId64 " \n", k, db->hash_tables[k].offsetB); - printf("checksumB[%lu]: %" PRIu64 " \n", k, db->hash_tables[k].checksumB); - printf("current [%lu]: %" PRIu64 " \n", k, db->hash_tables[k].current); + printf("ht[%d] offsetA [%lu]: %" PRId64 " \n",i, k, db->hashTables[i].slots[k].offsetA); + printf("ht[%d] offsetB [%lu]: %" PRId64 " \n",i, k, db->hashTables[i].slots[k].offsetB); + printf("ht[%d] current [%lu]: %" PRIu64 " \n",i, k, db->hashTables[i].slots[k].current); } } } #endif -void KISSDB_Iterator_init(KISSDB *db, KISSDB_Iterator *dbi) +void KISSDB_Iterator_init(KISSDB* db, KISSDB_Iterator* dbi) { dbi->db = db; dbi->h_no = 0; // number of read hashtables @@ -1277,235 +1306,953 @@ void KISSDB_Iterator_init(KISSDB *db, KISSDB_Iterator *dbi) } -int KISSDB_Iterator_next(KISSDB_Iterator *dbi, void *kbuf, void *vbuf) +int KISSDB_Iterator_next(KISSDB_Iterator* dbi, void* kbuf, void* vbuf) { + DataBlock_s* block; + Hashtable_slot_s* ht; int64_t offset; - Kdb_rdlock(&dbi->db->shmem_info->rwlock); - if ((dbi->h_no < (dbi->db->shmem_info->num_hash_tables)) && (dbi->h_idx < dbi->db->hash_table_size)) + if(dbi->db->htMappedSize < dbi->db->shared->htShmSize) { - //TODO check for currently valid data block flag and use this offset instead of offsetA - while (!(offset = dbi->db->hash_tables[((dbi->db->hash_table_size + 1) * dbi->h_no) + dbi->h_idx].offsetA)) + if ( Kdb_false == remapSharedHashtable(dbi->db->htFd, &dbi->db->hashTables, dbi->db->htMappedSize, dbi->db->shared->htShmSize)) + { + return KISSDB_ERROR_RESIZE_SHM; + } + else { - if (++dbi->h_idx >= dbi->db->hash_table_size) + dbi->db->htMappedSize = dbi->db->shared->htShmSize; + } + } + + //remap database file if in the meanwhile another process added new data (key value pairs / hashtables) to the file + if (dbi->db->dbMappedSize < dbi->db->shared->mappedDbSize) + { + dbi->db->mappedDb = mremap(dbi->db->mappedDb, dbi->db->dbMappedSize, dbi->db->shared->mappedDbSize, MREMAP_MAYMOVE); + if (dbi->db->mappedDb == MAP_FAILED) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(":mremap error: !"), DLT_STRING(strerror(errno))); + return KISSDB_ERROR_IO; + } + dbi->db->dbMappedSize = dbi->db->shared->mappedDbSize; + } + + if ((dbi->h_no < (dbi->db->shared->htNum)) && (dbi->h_idx < dbi->db->htSize)) + { + ht = dbi->db->hashTables[dbi->h_no].slots; //pointer to first hashtable + + while ( !(ht[dbi->h_idx].offsetA || ht[dbi->h_idx].offsetB) ) //until a offset was found + { + if (++dbi->h_idx >= dbi->db->htSize) { dbi->h_idx = 0; - if (++dbi->h_no >= (dbi->db->shmem_info->num_hash_tables)) + if (++dbi->h_no >= (dbi->db->shared->htNum)) { - Kdb_unlock(&dbi->db->shmem_info->rwlock); return 0; } + else + { + ht = dbi->db->hashTables[dbi->h_no].slots; //next hashtable + } } } - - if (lseek(dbi->db->fd, offset, SEEK_SET) == -1) - return KISSDB_ERROR_IO; - if (read(dbi->db->fd, kbuf, dbi->db->key_size) != dbi->db->key_size) - return KISSDB_ERROR_IO; - if (vbuf != NULL) + if(ht[dbi->h_idx].current == 0x00) { - if (read(dbi->db->fd, vbuf, dbi->db->value_size) != dbi->db->value_size) - return KISSDB_ERROR_IO; + offset = ht[dbi->h_idx].offsetA; } else { - if (lseek(dbi->db->fd, dbi->db->value_size, SEEK_CUR) == -1) - return KISSDB_ERROR_IO; + offset = ht[dbi->h_idx].offsetB; + } + + if( abs(offset) > dbi->db->dbMappedSize ) + { + return KISSDB_ERROR_IO; } - if (++dbi->h_idx >= dbi->db->hash_table_size) + block = (DataBlock_s*) (dbi->db->mappedDb + offset); + memcpy(kbuf,block->key, dbi->db->keySize); + if (vbuf != NULL) + { + memcpy(vbuf, block->value, dbi->db->valSize); + } + if (++dbi->h_idx >= dbi->db->htSize) { dbi->h_idx = 0; ++dbi->h_no; } - Kdb_unlock(&dbi->db->shmem_info->rwlock); return 1; } - Kdb_unlock(&dbi->db->shmem_info->rwlock); return 0; } -int readHeader(KISSDB* db, uint16_t* hash_table_size, uint64_t* key_size, uint64_t* value_size) + +int readHeader(KISSDB* db, uint16_t* htSize, uint64_t* keySize, uint64_t* valSize) { - //set Filepointer to the beginning of the file - if (lseek(db->fd, 0, SEEK_SET) == -1) - return KISSDB_ERROR_IO; - //mmap header from beginning of file - int mapFlag = PROT_WRITE | PROT_READ; Header_s* ptr = 0; - ptr = (Header_s*) mmap(NULL, KISSDB_HEADER_SIZE, mapFlag, MAP_SHARED, db->fd, 0); - if (ptr == MAP_FAILED) - return KISSDB_ERROR_IO; + ptr = (Header_s*) db->mappedDb; - if ((ptr->KdbV[0] != 'K') || (ptr->KdbV[1] != 'd') || (ptr->KdbV[2] != 'B') || (ptr->KdbV[3] != KISSDB_VERSION)) + if ((ptr->KdbV[0] != 'K') || (ptr->KdbV[1] != 'd') || (ptr->KdbV[2] != 'B') ) + { return KISSDB_ERROR_CORRUPT_DBFILE; - - if (!ptr->hash_table_size) + } + if( (ptr->KdbV[3] != KISSDB_MAJOR_VERSION) || (ptr->KdbV[4] != '.') || (ptr->KdbV[5] != KISSDB_MINOR_VERSION)) + { + return KISSDB_ERROR_WRONG_DATABASE_VERSION; + } + if (!ptr->htSize) + { return KISSDB_ERROR_CORRUPT_DBFILE; - (*hash_table_size) = (uint16_t) ptr->hash_table_size; - - if (!ptr->key_size) + } + (*htSize) = (uint16_t) ptr->htSize; + if (!ptr->keySize) + { return KISSDB_ERROR_CORRUPT_DBFILE; - (*key_size) = (uint64_t) ptr->key_size; + } + (*keySize) = (uint64_t) ptr->keySize; - if (!ptr->value_size) + if (!ptr->valSize) + { return KISSDB_ERROR_CORRUPT_DBFILE; - (*value_size) = (uint64_t) ptr->value_size; - - //sync changes with file - if (0 != msync(ptr, KISSDB_HEADER_SIZE, MS_SYNC | MS_INVALIDATE)) - return KISSDB_ERROR_IO; - - //unmap memory - if (0 != munmap(ptr, KISSDB_HEADER_SIZE)) - return KISSDB_ERROR_IO; + } + (*valSize) = (uint64_t) ptr->valSize; return 0; } -int writeHeader(KISSDB* db, uint16_t* hash_table_size, uint64_t* key_size, uint64_t* value_size) +int writeHeader(KISSDB* db, uint16_t* htSize, uint64_t* keySize, uint64_t* valSize) { Header_s* ptr = 0; int ret= 0; - //Seek to beginning of file - if (lseek(db->fd, 0, SEEK_SET) == -1) - return KISSDB_ERROR_IO; - - //ftruncate file to needed size for header + //truncate file to needed size for header ret = ftruncate(db->fd, KISSDB_HEADER_SIZE); if (ret < 0) + { return KISSDB_ERROR_IO; - - //mmap header from beginning of file - int mapFlag = PROT_WRITE | PROT_READ; - ptr = (Header_s*) mmap(NULL, KISSDB_HEADER_SIZE, mapFlag, MAP_SHARED, db->fd, 0); - if (ptr == MAP_FAILED) + } + //mmap whole file for the first time + db->mappedDb = (void*) mmap(NULL, KISSDB_HEADER_SIZE, PROT_WRITE | PROT_READ, MAP_SHARED, db->fd, 0); + if (db->mappedDb == MAP_FAILED) + { return KISSDB_ERROR_IO; + } + db->shared->mappedDbSize = KISSDB_HEADER_SIZE; + db->dbMappedSize = KISSDB_HEADER_SIZE; + ptr = (Header_s*) db->mappedDb; ptr->KdbV[0] = 'K'; ptr->KdbV[1] = 'd'; ptr->KdbV[2] = 'B'; - ptr->KdbV[3] = KISSDB_VERSION; - ptr->KdbV[4] = '-'; - ptr->KdbV[5] = '-'; - ptr->KdbV[6] = '-'; - ptr->KdbV[7] = '-'; + ptr->KdbV[3] = KISSDB_MAJOR_VERSION; + ptr->KdbV[4] = '.'; + ptr->KdbV[5] = KISSDB_MINOR_VERSION; + ptr->KdbV[6] = '0'; + ptr->KdbV[7] = '0'; ptr->checksum = 0x00; ptr->closeFailed = 0x00; //remove closeFailed flag ptr->closeOk = 0x01; //set closeOk flag - ptr->hash_table_size = (uint64_t)(*hash_table_size); - ptr->key_size = (uint64_t)(*key_size); - ptr->value_size = (uint64_t)(*value_size); - memcpy(ptr->delimiter,"||||||||", 8); + ptr->htSize = (uint64_t)(*htSize); + ptr->keySize = (uint64_t)(*keySize); + ptr->valSize = (uint64_t)(*valSize); + msync(db->mappedDb, KISSDB_HEADER_SIZE, MS_SYNC); - //sync changes with file - if (0 != msync(ptr, KISSDB_HEADER_SIZE, MS_SYNC | MS_INVALIDATE)) - return KISSDB_ERROR_IO; - - //unmap memory - if (0 != munmap(ptr, KISSDB_HEADER_SIZE)) - return KISSDB_ERROR_IO; return 0; } int checkErrorFlags(KISSDB* db) { - //mmap header from beginning of file - int mapFlag = PROT_WRITE | PROT_READ; - Header_s* ptr = 0; - ptr = (Header_s*) mmap(NULL, KISSDB_HEADER_SIZE, mapFlag, MAP_SHARED, db->fd, 0); - if (ptr == MAP_FAILED) - return KISSDB_ERROR_IO; - //uint64_t crc = 0; + Header_s* ptr; + ptr = (Header_s*) db->mappedDb; -#ifdef __checkerror - //check if closeFailed flag is set - if(ptr->closeFailed == 0x01) + //check if closeFailed flag is set or closeOk is not set + if(ptr->closeFailed == 0x01 || ptr->closeOk == 0x00 ) + { +#ifdef PFS_TEST + printf("CHECK ERROR FLAGS: CLOSE FAILED!\n"); +#endif + ptr->closeFailed = 0x01; //create close failed flags + ptr->closeOk = 0x00; + return KISSDB_ERROR_CORRUPT_DBFILE; + } + else { - //TODO implement verifyHashtableCS +#ifdef PFS_TEST + printf("CHECK ERROR FLAGS: CLOSE OK!\n"); +#endif + ptr->closeFailed = 0x01; //NO: create close failed flag + ptr->closeOk = 0x00; + } + msync(db->mappedDb, KISSDB_HEADER_SIZE, MS_SYNC); - //if closeFailed flag is set, something went wrong at last close -> so check crc - db->shmem_info->crc_invalid = Kdb_true; //check crc for further reads + return 0; +} -#if 0 - DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING("OPENING DB -> closeFailed flag is set: "), DLT_UINT64(ptr->closeFailed)); - crc = (uint64_t) pcoCalcCrc32Csum(db->fd, sizeof(Header_s)); - if(ptr->checksum != 0) //do not check if database is currently in creation + +int verifyHashtableCS(KISSDB* db) +{ + char* ptr; + Hashtable_s* hashtable; + int i = 0; + int ptrOffset=1; + int64_t offset = 0; + struct stat statBuf; + uint64_t crc = 0; + void* memory; + + if (db->fd) + { + //map entire file into memory + fstat(db->fd, &statBuf); + memory = (void*) mmap(NULL, statBuf.st_size, PROT_WRITE | PROT_READ, MAP_SHARED, db->fd, 0); + if (memory == MAP_FAILED) { - if (crc != ptr->checksum) + return KISSDB_ERROR_IO; + } + ptr = (char*) memory; + db->shared->htNum = 0; + //unmap previously allocated and maybe corrupted hashtables + munmap(db->hashTables, db->htMappedSize); + db->hashTables = (Hashtable_s*) getKdbShmemPtr(db->htFd, db->htSizeBytes); + if(db->hashTables == ((void*) -1)) + { + return KISSDB_ERROR_MAP_SHM; + } + db->htMappedSize = db->htSizeBytes; //size for first hashtable + + //determine greatest common factor of hashtable and datablock used for pointer incrementation + ptrOffset = greatestCommonFactor(sizeof(DataBlock_s), sizeof(Hashtable_s) ); + + //offsets in mapped area to first hashtable + offset = sizeof(Header_s); + ptr += offset; + + //get number of hashtables in file (search for hashtable delimiters) and copy the hashtables to memory + while (offset <= (statBuf.st_size - db->htSizeBytes)) //begin with offset for first hashtable + { + hashtable = (Hashtable_s*) ptr; + //if at least one of two hashtable delimiters are found + if (hashtable->delimStart == HASHTABLE_START_DELIMITER + || hashtable->delimEnd == HASHTABLE_END_DELIMITER) { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING("OPENING DB: "), DLT_STRING(db->shmem_ht_name), DLT_STRING(" CHECKSUM IN HEADER : "), DLT_UINT64(ptr->checksum), DLT_STRING(" != CHECKSUM CALCULATED: "), DLT_UINT64(crc)); - //db->shmem_info->crc_invalid = Kdb_true; //check datablocks at further reads - //return KISSDB_ERROR_CORRUPT_DBFILE; //previous close failed and checksum invalid -> error state -> return error + //next hashtable to use + //rewrite delimiters to make sure that both exist + hashtable->delimStart = HASHTABLE_START_DELIMITER; + hashtable->delimEnd = HASHTABLE_END_DELIMITER; + Kdb_bool result = Kdb_false; + //if new size would exceed old shared memory size-> allocate additional memory page to shared memory + if (db->htSizeBytes * (db->shared->htNum + 1) > db->htMappedSize) + { + result = resizeKdbShmem(db->htFd, &db->hashTables, db->htMappedSize, db->htMappedSize + db->htSizeBytes); + if (result == Kdb_false) + { + return KISSDB_ERROR_RESIZE_SHM; + } + else + { + db->shared->htShmSize = db->htMappedSize + db->htSizeBytes; + db->htMappedSize = db->shared->htShmSize; + } + } + // copy the current hashtable read from file to (htadress + (htsize * htcount)) in memory + memcpy(((uint8_t*) db->hashTables) + (db->htSizeBytes * db->shared->htNum), ptr, db->htSizeBytes); + ++db->shared->htNum; + + //jump to next data block after hashtable + offset += sizeof(Hashtable_s); + ptr += sizeof(Hashtable_s); } else { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("OPENING DB: "), DLT_STRING(db->shmem_ht_name), DLT_STRING(" CECHKSUM IN HEADER: "), DLT_UINT64(ptr->checksum), DLT_STRING(" == CHECKSUM CALCULATED: "), DLT_UINT64(crc)); - //db->shmem_info->crc_invalid = Kdb_false; //do not check datablocks at further reads + offset += ptrOffset; + ptr += ptrOffset; } } - else - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("Do not check checksum, database in creation: "), DLT_STRING(db->shmem_ht_name)); + munmap(memory, statBuf.st_size); + + //check CRC of all found hashtables + if (db->shared->htNum > 0) + { + for (i = 0; i < db->shared->htNum; i++) + { + crc = 0; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) db->hashTables[i].slots, sizeof(db->hashTables[i].slots)); + if (db->hashTables[i].crc != crc) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(": Checksum of hashtable number: <"); DLT_INT(i); DLT_STRING("> is invalid")); +#ifdef PFS_TEST + printf("VERIFY HASHTABLE: hashtable #%d: CHECKSUM INVALID! \n",i); #endif + return -1; //start rebuild of hashtables + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(__FUNCTION__); DLT_STRING(": Checksum of hashtable number: <"); DLT_INT(i); DLT_STRING("> is OK")); + +#ifdef PFS_TEST + printf("VERIFY HASHTABLE: hashtable #%d: CHECKSUM OK! \n",i); +#endif + } + } + } } - else + return 0; +} + +int rebuildHashtables(KISSDB* db) +{ + char* ptr; + DataBlock_s* data; + DataBlock_s* dataA; + DataBlock_s* dataB; + Hashtable_s* hashtable; + int current = 0; + int ptrOffset = 1; + int64_t offset=0; + int64_t offsetA = 0; + struct stat statBuf; + uint64_t crc = 0; + uint64_t calcCrcA, calcCrcB, readCrcA, readCrcB; + void* memory; + + fstat(db->fd, &statBuf); + memory = (char*) mmap(NULL, statBuf.st_size, PROT_WRITE | PROT_READ, MAP_SHARED, db->fd, 0); + if (memory == MAP_FAILED) { - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("OPENING DB: closeFailed flag is not set: "), DLT_UINT64(ptr->closeFailed)); - ptr->closeFailed = 0x01; //NO: create close failed flag + return KISSDB_ERROR_IO; } + ptr = (char*) memory; + + //recover all hashtables of database + if (db->shared->htNum > 0) //htNum was determined in verifyhashtables() -> no reallocation is needed + { + ptr = (void*) memory; + memset(db->hashTables, 0, db->shared->htNum * sizeof(Hashtable_s)); + db->hashTables[0].delimStart = HASHTABLE_START_DELIMITER; + db->hashTables[0].delimEnd = HASHTABLE_END_DELIMITER; + + ptrOffset = greatestCommonFactor(sizeof(DataBlock_s), sizeof(Hashtable_s) ); + + //begin searching after first hashtable + offset = sizeof(Header_s) + sizeof(Hashtable_s); + ptr += offset; + //go through database file until offset + Datablock size reaches end of file mapping + while (offset <= (statBuf.st_size - sizeof(DataBlock_s))) + { + data = (DataBlock_s*) ptr; - //check if closeOk flag is set - if(ptr->closeOk == 0x01) + //if block A start or end delimiters were found + if (data->delimStart == DATA_BLOCK_A_START_DELIMITER + || data->delimEnd == DATA_BLOCK_A_END_DELIMITER) + { + //calculate checksum of Block A + calcCrcA = 0; + calcCrcA = (uint64_t) pcoCrc32(calcCrcA, (unsigned char*) data->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); + readCrcA = data->crc; + + //search for block B start delimiter + offset += sizeof(DataBlock_s); + ptr += sizeof(DataBlock_s); + dataB = (DataBlock_s*) ptr; + if (dataB->delimStart == DATA_BLOCK_B_START_DELIMITER + || dataB->delimEnd == DATA_BLOCK_B_END_DELIMITER) + { + //verify checksum of Block B + calcCrcB = 0; + calcCrcB = (uint64_t) pcoCrc32(calcCrcB, (unsigned char*) dataB->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); + readCrcB = dataB->crc; + if (readCrcB == calcCrcB) //checksum of block B matches + { + if (readCrcA == calcCrcA) //checksum of block A matches + { + if (1) //decide which datablock has latest written data - still statically using Block B for recovery because both blocks are written in kissdb_put + { + offsetA = offset - sizeof(DataBlock_s); + rebuildWithBlockB(dataB, db, offsetA, offset); + } + else + { + // use block A for rebuild + //write offsets for block a and block B + offsetA = offset - sizeof(DataBlock_s); + rebuildWithBlockA(data, db, offsetA, offset); + } + } + else //checksum of block A does not match, but checksum of block B was valid + { + // use block B for rebuild + offsetA = offset - sizeof(DataBlock_s); + rebuildWithBlockB(dataB, db, offsetA, offset); + } + } + else + { + if (readCrcA == calcCrcA) + { + // use block A for rebuild + //write offsets for block a and block B + offsetA = offset - sizeof(DataBlock_s); + rebuildWithBlockA(data, db, offsetA, offset); + } + else //checksum of block A and of Block B do not match ---> worst case scenario + { + invalidateBlocks(data, dataB, db); + } + } + } + else + { + //if checksum A matches and block B was not found + if (readCrcA == calcCrcA) + { + // use block A for rebuild + //write offsets for block a and block B + offsetA = offset - sizeof(DataBlock_s); + rebuildWithBlockA(data, db, offsetA, offset); + } + else //checksum of block A does not match and block B was not found + { + invalidateBlocks(data, dataB, db); + } + } + //jump behind datablock B + offset += sizeof(DataBlock_s); + ptr += sizeof(DataBlock_s); + } + //If a Bock B start or end delimiters were found: this only can happen if previous Block A was not found + else if (data->delimStart == DATA_BLOCK_B_START_DELIMITER + || data->delimEnd == DATA_BLOCK_B_END_DELIMITER) + { + dataA = (DataBlock_s*) (ptr - sizeof(DataBlock_s)) ; + //verify checksum of Block B + crc = 0; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) data->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); + if (data->crc == crc) + { + //use block B for rebuild + //write offsets for block A and block B + offsetA = offset - sizeof(DataBlock_s); + rebuildWithBlockB(data, db, offsetA, offset); + } + else + { + invalidateBlocks(dataA, data, db); + } + //jump behind datablock B + offset += sizeof(DataBlock_s); + ptr += sizeof(DataBlock_s); + } + else if (data->delimStart == DATA_BLOCK_A_DELETED_START_DELIMITER + || data->delimEnd == DATA_BLOCK_A_DELETED_END_DELIMITER) + { + //calculate checksum of Block A + calcCrcA = 0; + calcCrcA = (uint64_t) pcoCrc32(calcCrcA, (unsigned char*) data->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); + readCrcA = data->crc; + + //search for block B start delimiter + offset += sizeof(DataBlock_s); + ptr += sizeof(DataBlock_s); + dataB = (DataBlock_s*) ptr; + + if (dataB->delimStart == DATA_BLOCK_B_DELETED_START_DELIMITER + || dataB->delimEnd == DATA_BLOCK_B_DELETED_END_DELIMITER) + { + //calculate checksum of Block B + calcCrcB = 0; + calcCrcB = (uint64_t) pcoCrc32(calcCrcB, (unsigned char*) dataB->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); + readCrcB = dataB->crc; + if (readCrcB == calcCrcB) //checksum of block B matches + { + offsetA = offset - sizeof(DataBlock_s); + invertBlockOffsets(data, db, offsetA, offset); + } + else + { + if (readCrcA == calcCrcA) + { + offsetA = offset - sizeof(DataBlock_s); + invertBlockOffsets(data, db, offsetA, offset); + } + else + { + invalidateBlocks(data, dataB, db); + } + } + } + else //NO BLOCK B Found + { + if (readCrcA == calcCrcA) + { + + offsetA = offset - sizeof(DataBlock_s); + invertBlockOffsets(data, db, offsetA, offset); + } + else + { + invalidateBlocks(data, dataB, db); + } + } + //jump behind datablock B + offset += sizeof(DataBlock_s); + ptr += sizeof(DataBlock_s); + } + else if (data->delimStart == DATA_BLOCK_B_DELETED_START_DELIMITER + || data->delimEnd == DATA_BLOCK_B_DELETED_END_DELIMITER) + { + crc = 0; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) data->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); + if (crc == data->crc) + { + offsetA = offset - sizeof(DataBlock_s); + invertBlockOffsets(data, db, offsetA, offset); + } + else + { + dataA = (DataBlock_s*) (ptr - sizeof(DataBlock_s)) ; + invalidateBlocks(dataA, data, db); + } + //jump behind datablock B + offset += sizeof(DataBlock_s); + ptr += sizeof(DataBlock_s); + } + else if( offset <= (statBuf.st_size - sizeof(Hashtable_s)) ) //check if ptr range for hashtable is within mapping of file + { + hashtable = (Hashtable_s*) ptr; + + if (hashtable->delimStart == HASHTABLE_START_DELIMITER + || hashtable->delimEnd == HASHTABLE_END_DELIMITER) + { + //next hashtable to use + db->hashTables[current].slots[db->htSize].offsetA = offset; //update link to next hashtable in current hashtable + current++; + if (current < db->shared->htNum) + { + db->hashTables[current].delimStart = HASHTABLE_START_DELIMITER; + db->hashTables[current].delimEnd = HASHTABLE_END_DELIMITER; + } + else + { + return -1; + } + offset += sizeof(Hashtable_s); + ptr += sizeof(Hashtable_s); + } + else //if no hashtable is found + { + offset += ptrOffset; + ptr += ptrOffset; + } + } + else //if nothing is found for offsets in -> (filesize - hashtablesize) area + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(": No Datablock or hashtable area found!")); + //increment pointer by greatest common factor of hashtable size and datablock size + offset += ptrOffset; + ptr += ptrOffset; + } + } + } + msync(memory, statBuf.st_size, MS_SYNC | MS_INVALIDATE); + munmap(memory, statBuf.st_size); + return 0; +} + + +//invalidate block content for A and B +//this block can never be found / overwritten again +//new insertions can reuse hashtable entry but block is added at EOF +void invalidateBlocks(DataBlock_s* dataA, DataBlock_s* dataB, KISSDB* db) +{ + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": Datablock recovery for key: <"); DLT_STRING(dataA->key); DLT_STRING("> impossible: both datablocks are invalid!")); + + memset(dataA->key, 0, db->keySize); + memset(dataA->value, 0, db->valSize); + dataA->crc=0; + dataA->htNum = 0; + dataA->valSize = 0; + + memset(dataB->key, 0, db->keySize); + memset(dataB->value, 0, db->valSize); + dataB->crc=0; + dataB->htNum = 0; + dataB->valSize = 0; +} + + +void invertBlockOffsets(DataBlock_s* data, KISSDB* db, int64_t offsetA, int64_t offsetB) +{ + uint64_t hash = 0; + hash = KISSDB_hash(data->key, strlen(data->key)) % (uint64_t) db->htSize; + //invert offsets for deleted block A + db->hashTables[data->htNum].slots[hash].offsetA = - offsetA; + //invert offsets for deleted block B + db->hashTables[data->htNum].slots[hash].offsetB = - offsetB; + //reset current flag + db->hashTables[data->htNum].slots[hash].current = 0x00; +} + + +void rebuildWithBlockB(DataBlock_s* data, KISSDB* db, int64_t offsetA, int64_t offsetB) +{ + uint64_t hash = KISSDB_hash(data->key, strlen(data->key)) % (uint64_t) db->htSize; + //write offsets for block A and block B + db->hashTables[data->htNum].slots[hash].offsetA = offsetA; + db->hashTables[data->htNum].slots[hash].offsetB = offsetB; + //set block B as current + db->hashTables[data->htNum].slots[hash].current = 0x01; + + /* + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(__FUNCTION__); DLT_STRING(": Rebuild in hashtable No. <"); DLT_INT(data->htNum); + DLT_STRING("> with Datablock B for key: <"); DLT_STRING(data->key); DLT_STRING("- hash: <"); DLT_INT(hash); DLT_STRING("> - OffsetA: <"); DLT_INT(offsetA); + DLT_STRING("> - OffsetB: <"); DLT_INT(offsetB)); + */ +} + + +void rebuildWithBlockA(DataBlock_s* data, KISSDB* db, int64_t offsetA, int64_t offsetB) +{ + uint64_t hash = KISSDB_hash(data->key, strlen(data->key)) % (uint64_t) db->htSize; + //write offsets for block A and block B + db->hashTables[data->htNum].slots[hash].offsetA = offsetA; + db->hashTables[data->htNum].slots[hash].offsetB = offsetB; + //set block B as current + db->hashTables[data->htNum].slots[hash].current = 0x00; + + /* + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(__FUNCTION__); DLT_STRING(": Rebuild in hashtable No. <"); DLT_INT(data->htNum); + DLT_STRING("> with Datablock A for key: <"); DLT_STRING(data->key); DLT_STRING("- hash: <"); DLT_INT(hash); DLT_STRING("> - OffsetA: <"); DLT_INT(offsetA); + DLT_STRING("> - OffsetB: <"); DLT_INT(offsetB)); + */ +} + + + + +int recoverDataBlocks(KISSDB* db) +{ + +#ifdef PFS_TEST + printf("DATABLOCK RECOVERY: START! \n"); +#endif + + char* ptr; + DataBlock_s* data; + int i = 0; + int k = 0; + int64_t offset = 0; + struct stat statBuf; + uint64_t crc = 0; + void* memory; + + fstat(db->fd, &statBuf); + memory = (char*) mmap(NULL, statBuf.st_size, PROT_WRITE | PROT_READ, MAP_SHARED, db->fd, 0); + if (memory == MAP_FAILED) { - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("OPENING DB -> closeOk flag is set: "), DLT_UINT64(ptr->closeOk)); - ptr->closeOk = 0x00; + return KISSDB_ERROR_IO; } - else + ptr = (char*) memory; + + //go through all hashtables and jump to data blocks for crc validation + if (db->shared->htNum > 0) + { + ptr = (void*) memory; + for (i = 0; i < db->shared->htNum; i++) + { + for (k = 0; k < HASHTABLE_SLOT_COUNT; k++) + { + if (db->hashTables[i].slots[k].offsetA > 0) //ignore deleted or unused slots + { + ptr = (void*) memory; //reset pointer + //current valid data is offset A or offset B? + offset = (db->hashTables[i].slots[k].current == 0x00) ? db->hashTables[i].slots[k].offsetA : db->hashTables[i].slots[k].offsetB; + ptr += offset; //set pointer to current valid datablock + data = (DataBlock_s*) ptr; + //check crc of data block marked as current in hashtable + crc = 0; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) data->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); + if (data->crc != crc) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING(__FUNCTION__); DLT_STRING(": Invalid datablock found at file offset: "); DLT_INT(offset)); +#ifdef PFS_TEST + printf("DATABLOCK RECOVERY: INVALID CRC FOR CURRENT DATABLOCK DETECTED! \n"); +#endif + ptr = (void*) memory; + //get offset to other data block and check crc + offset = (db->hashTables[i].slots[k].current == 0x00) ? db->hashTables[i].slots[k].offsetB : db->hashTables[i].slots[k].offsetA; + ptr += offset; + data = (DataBlock_s*) ptr; + crc = 0; + crc = (uint64_t) pcoCrc32(crc, (unsigned char*) data->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); + if (data->crc == crc) + { + //switch current flag if valid backup is available + db->hashTables[i].slots[k].current = (db->hashTables[i].slots[k].current == 0x00) ? 0x01 : 0x00; +#ifdef PFS_TEST + printf("DATABLOCK RECOVERY: REPAIR OF INVALID DATA SUCCESSFUL! \n"); +#endif + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(__FUNCTION__); DLT_STRING(": Invalid datablock for key: <"); DLT_STRING(data->key); DLT_STRING("> successfully recovered!")); + } + else + { + //invalidate data blocks if recovery fails + db->hashTables[i].slots[k].offsetA = - db->hashTables[i].slots[k].offsetA; + db->hashTables[i].slots[k].offsetB = - db->hashTables[i].slots[k].offsetB; + db->hashTables[i].slots[k].current = 0x00; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": Datablock recovery for key: <"); DLT_STRING(data->key); DLT_STRING("> impossible: both datablocks are invalid!")); +#ifdef PFS_TEST + printf("DATABLOCK RECOVERY: ERROR -> BOTH BLOCKS ARE INVALID! \n"); +#endif + } + } + } + } + } + } + msync(memory, statBuf.st_size, MS_SYNC | MS_INVALIDATE); + munmap(memory, statBuf.st_size); + +#ifdef PFS_TEST + printf("DATABLOCK RECOVERY: END! \n"); +#endif + + return 0; +} + + +int checkIsLink(const char* path, char* linkBuffer) +{ + char fileName[64] = { 0 }; + char truncPath[128] = { 0 }; + int isLink = 0; + int len = 0; + size_t strLen = 0; + struct stat statBuf; + uint16_t i = 0; + + memset(&statBuf, 0, sizeof(statBuf)); + strLen = strlen(path); + for (i = 0; i < strLen; i++) { - //if closeOK is not set , something went wrong at last close - db->shmem_info->crc_invalid = Kdb_true; //do crc check at read + if (path[i] == '/') + { + len = i; // remember the position of the last '/' + } + } + strncpy(truncPath, path, len); + truncPath[len + 1] = '\0'; // set end of string + strncpy(fileName, (const char*) path + len, 64); -#if 0 - crc = (uint64_t) pcoCalcCrc32Csum(db->fd, sizeof(Header_s)); - if(ptr->checksum != 0) //do not check if database is currently in creation + if (lstat(truncPath, &statBuf) != -1) + { + if (S_ISLNK(statBuf.st_mode)) { - if (crc != ptr->checksum) + if (readlink(truncPath, linkBuffer, 256) != -1) { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING("OPENING DB: "), DLT_STRING(db->shmem_ht_name), DLT_STRING(" CHECKSUM IN HEADER : "), DLT_UINT64(ptr->checksum), DLT_STRING(" != CHECKSUM CALCULATED: "), DLT_UINT64(crc)); - //db->shmem_info->crc_invalid = Kdb_true; - //return KISSDB_ERROR_CORRUPT_DBFILE; //previous close failed and checksum invalid -> error state -> return error + strncat(linkBuffer, fileName, 256); + isLink = 1; } else { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("OPENING DB: "), DLT_STRING(db->shmem_ht_name), DLT_STRING(" CECHKSUM IN HEADER: "), DLT_UINT64(ptr->checksum), DLT_STRING(" == CHECKSUM CALCULATED: "), DLT_UINT64(crc)); - //db->shmem_info->crc_invalid = Kdb_false; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(__FUNCTION__); DLT_STRING(": readlink failed: "); DLT_STRING(strerror(errno))); + isLink = -1; } } else { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, DLT_STRING("Do not check checksum, database in creation: "), DLT_STRING(db->shmem_ht_name)); + isLink = -1; } + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING(__FUNCTION__); DLT_STRING(": lstat failed: "); DLT_STRING(strerror(errno))); + isLink = -1; + } + return isLink; +} + - DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, DLT_STRING("OPENING DB -> closeOk flag is not set: "), DLT_UINT64(ptr->closeOk)); -#endif +void cleanKdbStruct(KISSDB* db) +{ + if (db->shared != NULL) + { + //Clean for every instance + if (db->mappedDb != NULL) + { + munmap(db->mappedDb, db->dbMappedSize); + db->mappedDb = NULL; + } + if (db->hashTables != NULL) + { + munmap(db->hashTables, db->htMappedSize); + db->hashTables = NULL; + } + if(db->cacheName != NULL) + { + free(db->cacheName); + db->cacheName = NULL; + } + if (db->fd) + { + close(db->fd); + db->fd = 0; + } + if(db->sharedCacheFd) + { + close(db->sharedCacheFd); + db->sharedCacheFd = -1; + } + db->alreadyOpen = Kdb_false; + db->htSize = 0; + //db->cacheReferenced = 0; + db->keySize = 0; + db->valSize = 0; + db->htSizeBytes = 0; + db->htMappedSize = 0; + db->dbMappedSize = 0; + db->shmCreator = 0; + + Kdb_unlock(&db->shared->rwlock); + + //Clean up for last instance referencing the database + if (db->shared->refCount == 0) + { + //close shared hashtable memory + if (db->htFd) + { + kdbShmemClose(db->htFd, db->htName); + db->htFd = 0; + } + if(db->htName != NULL) + { + free(db->htName); + db->htName = NULL; + } + //free rwlocks + pthread_rwlock_destroy(&db->shared->rwlock); + if (db->shared != NULL) + { + munmap(db->shared, sizeof(Shared_Data_s)); + db->shared = NULL; + } + if (db->sharedFd) + { + kdbShmemClose(db->sharedFd, db->sharedName); + db->sharedFd = 0; + } + if(db->sharedName != NULL) + { + free(db->sharedName); + db->sharedName = NULL; + } + //destroy and unlock named semaphore only if ref counter is zero + if (-1 == sem_post(db->kdbSem)) //release semaphore + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": sem_post() in cleanup failed: "), + DLT_STRING(strerror(errno))); + } + if (-1 == sem_close(db->kdbSem)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": sem_close() in cleanup failed: "), + DLT_STRING(strerror(errno))); + } + if (-1 == sem_unlink(db->semName)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": sem_unlink() in cleanup failed: "), + DLT_STRING(strerror(errno))); + } + db->kdbSem = NULL; + } + else //Clean up if other instances have reference to the database + { + if (db->sharedFd) + { + close(db->sharedFd); + db->sharedFd = 0; + } + if (db->htFd) + { + close(db->htFd); + db->htFd = 0; + } + if (db->shared != NULL) + { + munmap(db->shared, sizeof(Shared_Data_s)); + db->shared = NULL; + } + if(db->htName != NULL) + { + free(db->htName); + db->htName = NULL; + } + if (-1 == sem_post(db->kdbSem)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": sem_post() in cleanup with refcounter > 0 failed: "), + DLT_STRING(strerror(errno))); + } + if (-1 == sem_close(db->kdbSem)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": sem_close() in cleanup with refcounter > 0 failed: "), + DLT_STRING(strerror(errno))); + } + db->kdbSem = NULL; + } } -#endif - //sync changes with file - if (0 != msync(ptr, KISSDB_HEADER_SIZE, MS_SYNC | MS_INVALIDATE)) - return KISSDB_ERROR_IO; +} - //unmap memory - if (0 != munmap(ptr, KISSDB_HEADER_SIZE)) - return KISSDB_ERROR_IO; + +int writeDualDataBlock(KISSDB* db, int64_t offset, int htNumber, const void* key, unsigned long klen, const void* value, int valueSize) +{ + DataBlock_s* backupBlock; + DataBlock_s* block; + uint64_t crc = 0x00; + + block = (DataBlock_s*) (db->mappedDb + offset); + block->delimStart = DATA_BLOCK_A_START_DELIMITER; + memcpy(block->key,key, klen); + block->valSize = valueSize; + memcpy(block->value,value, block->valSize); + block->htNum = htNumber; + crc = 0x00; + crc = (uint32_t) pcoCrc32(crc, (unsigned char*)block->key, db->keySize + sizeof(uint32_t) + db->valSize + sizeof(uint64_t)); //crc over key, datasize, data and htnum + block->crc = crc; + block->delimEnd = DATA_BLOCK_A_END_DELIMITER; + + // write same key and value again + backupBlock = (DataBlock_s*) ((char*) block + sizeof(DataBlock_s)); + backupBlock->delimStart = DATA_BLOCK_B_START_DELIMITER; + backupBlock->crc = crc; + memcpy(backupBlock->key,key, klen); + backupBlock->valSize = valueSize; + memcpy(backupBlock->value,value, backupBlock->valSize); + backupBlock->htNum = htNumber; + backupBlock->delimEnd = DATA_BLOCK_B_END_DELIMITER; return 0; } + + +int greatestCommonFactor(int x, int y) +{ + while (y != 0) + { + if (x > y) + { + x = x - y; + } + else + { + y = y - x; + } + } + return x; +} diff --git a/src/key-value-store/database/kissdb.h b/src/key-value-store/database/kissdb.h index 90446ff..582e786 100644 --- a/src/key-value-store/database/kissdb.h +++ b/src/key-value-store/database/kissdb.h @@ -25,46 +25,68 @@ #include #include #include +#include #include "../hashtable/qlibc.h" +#include "../inc/protected/persComDbAccess.h" #ifdef __cplusplus extern "C" { #endif -//#define __showTimeMeasurements +/* #define __showTimeMeasurements */ + +#define DATA_BLOCK_A_START_DELIMITER 0x2AAAAAAA +#define DATA_BLOCK_A_END_DELIMITER 0x55555555 + +#define DATA_BLOCK_B_START_DELIMITER 0xE38E38E3 +#define DATA_BLOCK_B_END_DELIMITER 0xAAAAAAA8 + +#define DATA_BLOCK_A_DELETED_START_DELIMITER 0xAAAAAAAA +#define DATA_BLOCK_A_DELETED_END_DELIMITER 0xD5555555 + +#define DATA_BLOCK_B_DELETED_START_DELIMITER 0x7E07E07E +#define DATA_BLOCK_B_DELETED_END_DELIMITER 0x81F81F81 + +#define HASHTABLE_START_DELIMITER 0x33333333 +#define HASHTABLE_END_DELIMITER 0xCCCCCCCC + +#define HASHTABLE_SLOT_COUNT 510 #ifdef __showTimeMeasurements #define SECONDS2NANO 1000000000L #define NANO2MIL 1000000L #define MIL2SEC 1000L -// define for the used clock: "CLOCK_MONOTONIC" or "CLOCK_REALTIME" +/* define for the used clock: "CLOCK_MONOTONIC" or "CLOCK_REALTIME" */ #define CLOCK_ID CLOCK_MONOTONIC #endif /** - * Version: 2 + * Version: 2.3 * * This is the file format identifier, and changes any time the file - * format changes. The code version will be this dot something, and can - * be seen in tags in the git repository. + * format changes. */ -#define KISSDB_VERSION 2 +#define KISSDB_MAJOR_VERSION 2 +#define KISSDB_MINOR_VERSION 3 -//boolean type typedef int16_t Kdb_bool; static const int16_t Kdb_true = -1; static const int16_t Kdb_false = 0; typedef struct { - uint64_t shmem_size; - uint16_t num_hash_tables; - Kdb_bool cache_initialised; - Kdb_bool crc_invalid; + uint64_t htShmSize; /* shared info about current size of hashtable shared memory */ + /*uint64_t cacheSize;*/ + /*uint16_t cacheCount;*/ + uint16_t htNum; + uint16_t refCount; + uint16_t openMode; + uint16_t writeMode; + Kdb_bool cacheCreated; /* flag to indicate if the shared cache was created */ pthread_rwlock_t rwlock; - pthread_rwlock_t cache_rwlock; + uint64_t mappedDbSize; /* shared information about current mapped size of database file */ } Shared_Data_s; @@ -74,30 +96,49 @@ typedef struct typedef struct { char KdbV[8]; - uint64_t checksum; // checksum over database file + uint64_t checksum; /* checksum over database file */ uint64_t closeFailed; uint64_t closeOk; - uint64_t hash_table_size; - uint64_t key_size; - uint64_t value_size; + uint64_t htSize; + uint64_t keySize; + uint64_t valSize; char delimiter[8]; + char padding[4032]; /* TODO remove padding*/ } Header_s; +typedef struct +{ + int64_t delimStart; + uint64_t crc; + char key[PERS_DB_MAX_LENGTH_KEY_NAME]; + uint32_t valSize; + char value[PERS_DB_MAX_SIZE_KEY_DATA]; /* 8028, 12124, 16220 -- > PERS_DB_MAX_SIZE_KEY_DATA = (pagesize * n) - (PERS_DB_MAX_LENGTH_KEY_NAME + 36) */ + uint64_t htNum; /*index which hashtable stores the offset for this data block */ + int64_t delimEnd; +} DataBlock_s; /** - * Hashtable slot entry -> same size for all struct members because of alignment problems on target system!! + * Hashtable slot entry -for usage with mmap -> 24 byte --> use 510 + 1 slots */ typedef struct { int64_t offsetA; - uint64_t checksumA; int64_t offsetB; - uint64_t checksumB; uint64_t current; //flag which offset points to the current data -> (if 0x00 offsetA points to current data, if 0x01 offsetB) } Hashtable_slot_s; +//hashtable structure (size is multiple of 4096 byte for usage in shared memory) +typedef struct +{ + int64_t delimStart; + uint64_t crc; + Hashtable_slot_s slots[HASHTABLE_SLOT_COUNT + 1]; //510 + 1 item (link to next hashtable) -> 12264 byte + int64_t delimEnd; +} Hashtable_s; //12288 byte -> 3 pages of 4096 byte + + /** * KISSDB database @@ -105,23 +146,28 @@ typedef struct * These fields should never be changed. */ typedef struct { - uint16_t hash_table_size; - uint64_t key_size; - uint64_t value_size; - uint64_t hash_table_size_bytes; - uint64_t old_mapped_size; - Kdb_bool shmem_creator; - Kdb_bool already_open; - Hashtable_slot_s *hash_tables; //shared: stores the hashtables - void* shmem_cached; //shared: memory for key-value pair caching - int shmem_info_fd; - int shmem_ht_fd; - int shmem_cached_fd; - char* shmem_info_name; - char* shmem_cached_name; - char* shmem_ht_name; - Shared_Data_s* shmem_info; - qhasharr_t *tbl; //reference to cached datastructure + uint16_t htSize; + //uint16_t cacheReferenced; + uint64_t keySize; + uint64_t valSize; + uint64_t htSizeBytes; + uint64_t htMappedSize; //local info about currently mapped hashtable size for this process + uint64_t dbMappedSize; //local info about currently mapped database size for this process + Kdb_bool shmCreator; //local information if this instance is the creator of the shared memory + Kdb_bool alreadyOpen; + Hashtable_s* hashTables; //local pointer to hashtables in shared memory + char* mappedDb; // local mapping of database file for every process + void* sharedCache; //shared: memory for key-value pair caching + int sharedFd; + int htFd; + int sharedCacheFd; + char* semName; + char* sharedName; + char* cacheName; + char* htName; + Shared_Data_s* shared; + qhasharr_t *tbl[1]; //reference to cache + sem_t* kdbSem; int fd; //local fd } KISSDB; @@ -151,11 +197,6 @@ typedef struct { */ #define KISSDB_ERROR_ACCESS_VIOLATION -5 -/** - * Unable to unmap shared memory - */ -#define KISSDB_ERROR_UNMAP_SHM -6 - /** * Unable to open shared memory */ @@ -181,6 +222,16 @@ typedef struct { */ #define KISSDB_ERROR_CLOSE_SHM -11 +/** + * try to open database with wrong version + */ +#define KISSDB_ERROR_WRONG_DATABASE_VERSION -12 + +/** + * buffer where data should be returned is too small + */ +#define KISSDB_ERROR_WRONG_BUFSIZE -13 + /** * Open mode: read only */ @@ -201,6 +252,18 @@ typedef struct { */ #define KISSDB_OPEN_MODE_RWREPLACE 4 +/** + * Write mode: cache is not used, data is directly written (writeThrough mode) + */ +#define KISSDB_WRITE_MODE_WT 5 + +/** + * Write mode: cache is used for database access + */ +#define KISSDB_WRITE_MODE_WC 6 + + + /** * Open database * @@ -212,7 +275,8 @@ typedef struct { * * @param db Database struct * @param path Path to file - * @param mode One of the KISSDB_OPEN_MODE constants + * @param openMode One of the KISSDB_OPEN_MODE constants + * @param writeMode One of the KISSDB_WRITE constants * @param hash_table_size Size of hash table in entries (must be >0) * @param key_size Size of keys in bytes * @param value_size Size of values in bytes @@ -221,7 +285,8 @@ typedef struct { extern int KISSDB_open( KISSDB *db, const char *path, - int mode, + int openMode, + int writeMode, uint16_t hash_table_size, uint64_t key_size, uint64_t value_size); @@ -242,7 +307,7 @@ extern int KISSDB_close(KISSDB *db); * @param vbuf Value buffer (value_size bytes capacity) * @return negative on error (see kissdb.h for error codes), 0 on success, 1 if key not found */ -extern int KISSDB_get(KISSDB *db,const void *key,void *vbuf); +extern int KISSDB_get(KISSDB *db,const void *key,void *vbuf, uint32_t bufsize, uint32_t* vsize); @@ -253,7 +318,7 @@ extern int KISSDB_get(KISSDB *db,const void *key,void *vbuf); * @param key Key (key_size bytes) * @return negative on error (see kissdb.h for error codes), 0 on success, 1 if key not found */ -extern int KISSDB_delete(KISSDB *db,const void *key); +extern int KISSDB_delete(KISSDB *db,const void *key, int32_t* bytesDeleted); /** * Put an entry (overwriting it if it already exists) @@ -266,7 +331,7 @@ extern int KISSDB_delete(KISSDB *db,const void *key); * @param value Value (value_size bytes) * @return negative on error (see kissdb.h for error codes) error, 0 on success */ -extern int KISSDB_put(KISSDB *db,const void *key,const void *value); +extern int KISSDB_put(KISSDB *db,const void *key,const void *value, int valueSize, int32_t* bytesWritten); /** * Cursor used for iterating over all entries in database @@ -297,8 +362,6 @@ extern void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi); * @return 0 if there are no more entries, negative on error, positive if kbuf/vbuf have been filled */ extern int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf); - - extern Kdb_bool freeKdbShmemPtr(void * shmem_ptr, size_t length); extern void * getKdbShmemPtr(int shmem, size_t length); extern Kdb_bool kdbShmemClose(int shmem, const char * shmName); @@ -307,9 +370,20 @@ extern char * kdbGetShmName(const char * format, const char * path); extern void Kdb_wrlock(pthread_rwlock_t * wrlock); extern void Kdb_rdlock(pthread_rwlock_t * rdlock); extern void Kdb_unlock(pthread_rwlock_t * lock); -extern int readHeader(KISSDB* db, uint16_t* hash_table_size, uint64_t* key_size, uint64_t* value_size); -extern int writeHeader(KISSDB* db, uint16_t* hash_table_size, uint64_t* key_size, uint64_t* value_size); +extern int readHeader(KISSDB* db, uint16_t* htSize, uint64_t* keySize, uint64_t* valSize); +extern int writeHeader(KISSDB* db, uint16_t* htSize, uint64_t* keySize, uint64_t* valSize); +extern int writeDualDataBlock(KISSDB* db, int64_t offset, int htNumber, const void* key, unsigned long klen, const void* value, int valueSize); extern int checkErrorFlags(KISSDB* db); +extern int verifyHashtableCS(KISSDB* db); +extern int rebuildHashtables(KISSDB* db); +extern int greatestCommonFactor(int x, int y); +extern void invalidateBlocks(DataBlock_s* dataA, DataBlock_s* dataB, KISSDB* db); +extern void invertBlockOffsets(DataBlock_s* data, KISSDB* db, int64_t offsetA, int64_t offsetB); +extern void rebuildWithBlockB(DataBlock_s* data, KISSDB* db, int64_t offsetA, int64_t offsetB); +extern void rebuildWithBlockA(DataBlock_s* data, KISSDB* db, int64_t offsetA, int64_t offsetB); +extern int recoverDataBlocks(KISSDB* db); +extern int checkIsLink(const char* path, char* linkBuffer); +extern void cleanKdbStruct(KISSDB* db); #if 0 extern void printSharedHashtable(KISSDB *db); @@ -319,5 +393,5 @@ extern void printSharedHashtable(KISSDB *db); } #endif -#endif +#endif //___KISSDB_H diff --git a/src/key-value-store/hashtable/qhash.h b/src/key-value-store/hashtable/qhash.h index b873799..027bc60 100644 --- a/src/key-value-store/hashtable/qhash.h +++ b/src/key-value-store/hashtable/qhash.h @@ -48,14 +48,7 @@ extern "C" { #endif extern bool qhashmd5(const void *data, size_t nbytes, void *retbuf); -//extern bool qhashmd5_file(const char *filepath, off_t offset, ssize_t nbytes, -// void *retbuf); - -//extern uint32_t qhashfnv1_32(const void *data, size_t nbytes); -//extern uint64_t qhashfnv1_64(const void *data, size_t nbytes); - extern uint32_t qhashmurmur3_32(const void *data, size_t nbytes); -//extern bool qhashmurmur3_128(const void *data, size_t nbytes, void *retbuf); #ifdef __cplusplus } diff --git a/src/key-value-store/hashtable/qhasharr.c b/src/key-value-store/hashtable/qhasharr.c index 97169bf..3602e84 100644 --- a/src/key-value-store/hashtable/qhasharr.c +++ b/src/key-value-store/hashtable/qhasharr.c @@ -150,6 +150,7 @@ #include "qhash.h" #include "qhasharr.h" + #ifndef _DOXYGEN_SKIP static bool put(qhasharr_t *tbl, const char *key, const void *value, @@ -165,6 +166,8 @@ static int size(qhasharr_t *tbl, int *maxslots, int *usedslots); static void free_(qhasharr_t *tbl); + + // internal usages static int _find_empty(qhasharr_t *tbl, int startidx); static int _get_idx(qhasharr_t *tbl, const char *key, unsigned int hash); @@ -217,56 +220,67 @@ size_t qhasharr_calculate_memsize(int max) { * qhasharr_t *tbl2 = qhasharr(memory, 0); * @endcode */ -qhasharr_t *qhasharr(void *memory, size_t memsize) { - // Structure memory. - qhasharr_data_t *data = (qhasharr_data_t *) memory; - - // Initialize data if memsize is set or use existing data. - if (memsize > 0) { - // calculate max - int maxslots = (memsize - sizeof(qhasharr_data_t)) - / sizeof(qhasharr_slot_t); - if (maxslots < 1 || memsize <= sizeof(qhasharr_t)) { - //errno = EINVAL; - return NULL; - } - - // Set memory. - //memset((void *) data, 0, memsize); //TODO check if initialisation is needed - data->maxslots = maxslots; - data->usedslots = 0; - data->num = 0; - } - - // Set data address. Shared memory returns virtul address. - data->slots = (qhasharr_slot_t *) (memory + sizeof(qhasharr_data_t)); - - // Create the table object. - qhasharr_t *tbl = (qhasharr_t *) malloc(sizeof(qhasharr_t)); - if (tbl == NULL) { - //errno = ENOMEM; - return NULL; - } - memset((void *) tbl, 0, sizeof(qhasharr_t)); - - // assign methods - tbl->put = put; - - tbl->get = get; - - tbl->getnext = getnext; - - tbl->remove = remove_; - - tbl->size = size; +qhasharr_t *qhasharr(void *memory, size_t memsize) +{ +// Structure memory. + qhasharr_data_t *data = (qhasharr_data_t *) memory; + + //printf("qhasharr memory pointer: %p \n", memory); +// Initialize data if memsize is set or use existing data. + if (memsize > 0) + { +// calculate max + int maxslots = (memsize - sizeof(qhasharr_data_t)) / sizeof(qhasharr_slot_t); + if (maxslots < 1 || memsize <= sizeof(qhasharr_t)) + { + errno = EINVAL; + return NULL; + } +// Set memory. + memset((void *) data, 0, memsize); + data->maxslots = maxslots; + data->usedslots = 0; + data->num = 0; + } + //printf("Memory address: %p, sizeof(qhasharr_data_t): %d \n", memory, sizeof(qhasharr_data_t)); +// Set data address. Shared memory returns virtul address. + data->slots = (qhasharr_slot_t *) (memory + sizeof(qhasharr_data_t)); + +// Create the table object. + qhasharr_t *tbl = (qhasharr_t *) malloc(sizeof(qhasharr_t)); + if (tbl == NULL) + { + errno = ENOMEM; + return NULL; + } + memset((void *) tbl, 0, sizeof(qhasharr_t)); +// assign methods + tbl->put = put; + tbl->get = get; + tbl->getnext = getnext; + tbl->remove = remove_; + tbl->size = size; + tbl->free = free_; + tbl->data = data; + return tbl; +} - tbl->free = free_; - tbl->data = data; - return tbl; +/** + * setMemoryAddress(): resets the mapped shared memory pointer. + * + * @param memory pointer to shared memory. + * @param tbl qhasharr_t container pointer + */ +void setMemoryAddress(void* memory, qhasharr_t *tbl ) +{ + qhasharr_data_t *data = (qhasharr_data_t *) memory; + data->slots = (qhasharr_slot_t *) (memory + sizeof(qhasharr_data_t)); + tbl->data = data; } + /** * qhasharr->put(): Put an object into this table. * @@ -284,19 +298,21 @@ qhasharr_t *qhasharr(void *memory, size_t memsize) { static bool put(qhasharr_t *tbl, const char *key, const void *value, size_t size) { if (tbl == NULL || key == NULL || value == NULL) { - //errno = EINVAL; + errno = EINVAL; return false; } qhasharr_data_t *data = tbl->data; - + //printf("put data-> ptr= %p ---- MAXSLOTS = %d \n", data, data->maxslots); // check full if (data->usedslots >= data->maxslots || ((data->usedslots + (size / 32)) >= data->maxslots)) { //DEBUG("hasharr: put %s - FULL", key); - //errno = ENOBUFS; + errno = ENOBUFS; + //printf("CACHE TOO SMALL \n"); return false; } + // get hash integer unsigned int hash = qhashmurmur3_32(key, strlen(key)) % data->maxslots; @@ -314,13 +330,12 @@ static bool put(qhasharr_t *tbl, const char *key, const void *value, if (idx >= 0) { // same key // remove and recall remove_(tbl, key); - //printf("overwriting existent key 2%s\n", key); return put(tbl, key, value, size); } else { // no same key, just hash collision // find empty slot int idx = _find_empty(tbl, hash); if (idx < 0) { - //errno = ENOBUFS; + errno = ENOBUFS; return false; } @@ -339,11 +354,10 @@ static bool put(qhasharr_t *tbl, const char *key, const void *value, } else { // in case of -1 or -2, move it. -1 used for collision resolution, // -2 used for oversized value data. - // find empty slot int idx = _find_empty(tbl, hash + 1); if (idx < 0) { - //errno = ENOBUFS; + errno = ENOBUFS; return false; } @@ -396,18 +410,14 @@ static void *get(qhasharr_t *tbl, const char *key, size_t *size) { //errno = EINVAL; return NULL; } - qhasharr_data_t *data = tbl->data; - // get hash integer unsigned int hash = qhashmurmur3_32(key, strlen(key)) % data->maxslots; - int idx = _get_idx(tbl, key, hash); if (idx < 0) { //errno = ENOENT; return NULL; } - return _get_data(tbl, idx, size); } @@ -446,12 +456,10 @@ static bool getnext(qhasharr_t *tbl, qnobj_t *obj, int *idx) { } qhasharr_data_t *data = tbl->data; - for (; *idx < data->maxslots; (*idx)++) { if (data->slots[*idx].count == 0 || data->slots[*idx].count == -2) { continue; } - size_t keylen = data->slots[*idx].data.pair.keylen; if (keylen > _Q_HASHARR_KEYSIZE) keylen = _Q_HASHARR_KEYSIZE; @@ -580,7 +588,9 @@ static int size(qhasharr_t *tbl, int *maxslots, int *usedslots) { if (maxslots != NULL) *maxslots = data->maxslots; if (usedslots != NULL) + { *usedslots = data->usedslots; + } return data->num; } @@ -669,7 +679,6 @@ static int _get_idx(qhasharr_t *tbl, const char *key, unsigned int hash) { continue; } } - return -1; } @@ -678,7 +687,6 @@ static void *_get_data(qhasharr_t *tbl, int idx, size_t *size) { //errno = ENOENT; return NULL; } - qhasharr_data_t *data = tbl->data; int newidx; @@ -753,7 +761,7 @@ static bool _put_data(qhasharr_t *tbl, int idx, unsigned int hash, if (tmpidx < 0) { //DEBUG("hasharr: Can't expand slot for key %s.", key); _remove_data(tbl, idx); - //errno = ENOBUFS; + errno = ENOBUFS; return false; } @@ -818,7 +826,6 @@ static bool _copy_slot(qhasharr_t *tbl, int idx1, int idx2) { // increase used slot counter data->usedslots++; - return true; } @@ -837,7 +844,6 @@ static bool _remove_slot(qhasharr_t *tbl, int idx) // decrease used slot counter data->usedslots--; - return true; } diff --git a/src/key-value-store/hashtable/qhasharr.h b/src/key-value-store/hashtable/qhasharr.h index 37d52f3..2c00977 100644 --- a/src/key-value-store/hashtable/qhasharr.h +++ b/src/key-value-store/hashtable/qhasharr.h @@ -54,7 +54,7 @@ extern "C" { #define _Q_HASHARR_VALUESIZE (32) /*!< knob for maximum data size in a slot. */ #define PERS_CACHE_MAX_SLOTS 100000 /**< Max. number of slots in the cache */ -#define PERS_CACHE_MEMSIZE (sizeof(qhasharr_data_t)+ (sizeof(qhasharr_slot_t) * (PERS_CACHE_MAX_SLOTS))) //approximately 2 MB +#define PERS_CACHE_MEMSIZE (sizeof(qhasharr_data_t)+ (sizeof(qhasharr_slot_t) * (PERS_CACHE_MAX_SLOTS))) /* types */ typedef struct qhasharr_slot_s qhasharr_slot_t; @@ -64,7 +64,7 @@ typedef struct qhasharr_s qhasharr_t; /* public functions */ extern qhasharr_t *qhasharr(void *memory, size_t memsize); extern size_t qhasharr_calculate_memsize(int max); - +extern void setMemoryAddress(void* memory, qhasharr_t *tbl); /** * qhasharr internal data slot structure */ diff --git a/src/key-value-store/pers_low_level_db_access.c b/src/key-value-store/pers_low_level_db_access.c index 1ca2b27..71f49ce 100644 --- a/src/key-value-store/pers_low_level_db_access.c +++ b/src/key-value-store/pers_low_level_db_access.c @@ -1,4 +1,4 @@ - /****************************************************************************** +/****************************************************************************** * Project Persistency * (c) copyright 2014 * Company XS Embedded GmbH @@ -7,21 +7,21 @@ * This Source Code Form is subject to the terms of the * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -******************************************************************************/ - /** + ******************************************************************************/ +/** * @file pers_low_level_db_access.c * @author Simon Disch * @brief Implementation of persComDbAccess.h * @see */ - #include #include #include #include #include #include +#include #include #include "./database/kissdb.h" #include "./hashtable/qlibc.h" @@ -32,20 +32,40 @@ #include "persComDbAccess.h" #include "persComRct.h" #include "pers_low_level_db_access_if.h" -#include "dlt.h" +#include #include #include +#include + + +/* #define PFS_TEST */ + /* L&T context */ #define LT_HDR "[persComLLDB]" +DltContext persComLldbDLTCtx; +DLT_DECLARE_CONTEXT (persComLldbDLTCtx) + + + +///// library constructor +//void pco_library_init(void) __attribute__((constructor)); +//void pco_library_init() { +// printf ("\n pco_library_init() constructor \n"); +//} +// +///// library destructor +//void pco_library_destroy(void) __attribute__((destructor)); +//void pco_library_destroy() { +// printf ("\n pco_library_destroy() destructor \n"); +//} -DLT_DECLARE_CONTEXT(persComLldbDLTCtx); /* ---------------------- local definition ---------------------------- */ /* max number of open handlers per process */ #define PERS_LLDB_NO_OF_STATIC_HANDLES 16 #define PERS_LLDB_MAX_STATIC_HANDLES (PERS_LLDB_NO_OF_STATIC_HANDLES-1) -#define PERS_STATUS_KEY_NOT_IN_CACHE -10 //!< key not in cache +#define PERS_STATUS_KEY_NOT_IN_CACHE -10 /* /!< key not in cache */ typedef struct { @@ -56,7 +76,7 @@ typedef struct typedef enum pers_lldb_cache_flag_e { CachedDataDelete = 0, /* Resource-Configuration-Table */ - CachedDataWrite, /* Local/Shared DB */ + CachedDataWrite /* Local/Shared DB */ } pers_lldb_cache_flag_e; typedef struct @@ -85,7 +105,7 @@ typedef struct typedef struct lldb_handles_list_el_s_ { lldb_handler_s sHandle; - struct lldb_handles_list_el_s_ * pNext; + struct lldb_handles_list_el_s_* pNext; } lldb_handles_list_el_s; typedef struct @@ -98,7 +118,8 @@ typedef struct static const char ListItemsSeparator = '\0'; /* shared by all the threads within a process */ -static lldb_handlers_s g_sHandlers = { { { 0 } } }; +static lldb_handlers_s g_sHandlers; // initialize to 0 and NULL +//static lldb_handlers_s g_sHandlers = { { { 0 } } }; static pthread_mutex_t g_mutexLldb = PTHREAD_MUTEX_INITIALIZER; /* ---------------------- local macros --------------------------------- */ @@ -112,24 +133,26 @@ static sint_t GetKeySizeFromKissLocalDB(sint_t dbHandler, pconststr_t key); static sint_t GetDataFromKissLocalDB(sint_t dbHandler, pconststr_t key, pstr_t buffer_out, sint_t bufSize); static sint_t GetDataFromKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s* pConfig); static sint_t SetDataInKissLocalDB(sint_t dbHandler, pconststr_t key, pconststr_t data, sint_t dataSize); -static sint_t SetDataInKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s const * pConfig); +static sint_t SetDataInKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s const* pConfig); static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler); static sint_t writeBackKissRCT(KISSDB* db, lldb_handler_s* pLldbHandler); static sint_t getListandSize(KISSDB* db, pstr_t buffer, sint_t size, bool_t bOnlySizeNeeded, pers_lldb_purpose_e purpose); -static sint_t putToCache(KISSDB* db, sint_t dataSize, char* tmp_key, void* insert_cached_data); -static sint_t getFromCache(KISSDB* db, void* tmp_key, void* readBuffer, sint_t bufsize, bool_t sizeOnly); -static sint_t getFromDatabaseFile(KISSDB* db, void* tmp_key, void* readBuffer, pers_lldb_purpose_e purpose, sint_t bufsize, bool_t sizeOnly); +static sint_t putToCache(KISSDB* db, sint_t dataSize, char* metaKey, void* cachedData); +static sint_t deleteFromCache(KISSDB* db, char* metaKey); +static sint_t getFromCache(KISSDB* db, void* metaKey, void* readBuffer, sint_t bufsize, bool_t sizeOnly); +static sint_t getFromDatabaseFile(KISSDB* db, void* metaKey, void* readBuffer, sint_t bufsize); /* access to resources shared by the threads within a process */ static bool_t lldb_handles_Lock(void); static bool_t lldb_handles_Unlock(void); static lldb_handler_s* lldb_handles_FindInUseHandle(sint_t dbHandler); static lldb_handler_s* lldb_handles_FindAvailableHandle(void); -static void lldb_handles_InitHandle(lldb_handler_s* psHandle_inout, pers_lldb_purpose_e ePurpose, str_t const * dbPathname); +static void lldb_handles_InitHandle(lldb_handler_s* psHandle_inout, pers_lldb_purpose_e ePurpose, str_t const* dbPathname); static bool_t lldb_handles_DeinitHandle(sint_t dbHandler); static int createCache(KISSDB* db); static int openCache(KISSDB* db); +//static int addCache(KISSDB* db); static int closeCache(KISSDB* db); /** @@ -142,15 +165,22 @@ static int closeCache(KISSDB* db); * * \return >=0 for success, negative value otherway (see pers_error_codes.h) */ -sint_t pers_lldb_open(str_t const * dbPathname, pers_lldb_purpose_e ePurpose, bool_t bForceCreationIfNotPresent) +sint_t pers_lldb_open(str_t const* dbPathname, pers_lldb_purpose_e ePurpose, bool_t bForceCreationIfNotPresent) { - sint_t returnValue = PERS_COM_FAILURE; bool_t bCanContinue = true; - lldb_handler_s* pLldbHandler = NIL; bool_t bLocked = false; - int mode = KISSDB_OPEN_MODE_RDWR; + char linkBuffer[256] = { 0 }; + const char* path; + int error = 0; + int kdbState = 0; + int openMode = KISSDB_OPEN_MODE_RDWR; //default is open existing in RDWR + int writeMode = KISSDB_WRITE_MODE_WC; //default is write cached + lldb_handler_s* pLldbHandler = NIL; + sint_t returnValue = PERS_COM_FAILURE; static bool_t bFirstCall = true; + path = dbPathname; + if (bFirstCall) { pid_t pid = getpid(); @@ -161,12 +191,14 @@ sint_t pers_lldb_open(str_t const * dbPathname, pers_lldb_purpose_e ePurpose, bo /* init DLT */ (void) snprintf(dltContextID, sizeof(dltContextID), "Pers_%04d", pid); DLT_REGISTER_CONTEXT(persComLldbDLTCtx, dltContextID, "PersCommonLLDB"); - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("register context PersCommonLLDB ContextID="); DLT_STRING(dltContextID)); + //DLT_SET_APPLICATION_LL_TS_LIMIT(DLT_LOG_DEBUG, DLT_TRACE_STATUS_OFF); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("register context PersCommonLLDB ContextID="); DLT_STRING(dltContextID)); } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING("Begin opening:"); DLT_STRING("<<"); DLT_STRING(dbPathname); DLT_STRING(">>, "); ((PersLldbPurpose_RCT == ePurpose) ? DLT_STRING("RCT, ") : DLT_STRING("DB, ")); ((true == bForceCreationIfNotPresent) ? DLT_STRING("forced, ") : DLT_STRING("unforced, ")); DLT_STRING(" ... ")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING("Begin opening:"); DLT_STRING("<"); DLT_STRING(dbPathname); DLT_STRING(">, "); + ((PersLldbPurpose_RCT == ePurpose) ? DLT_STRING("RCT, ") : DLT_STRING("DB, ")); ((true == bForceCreationIfNotPresent) ? DLT_STRING("forced, ") : DLT_STRING("unforced, "))); if (lldb_handles_Lock()) { @@ -179,58 +211,158 @@ sint_t pers_lldb_open(str_t const * dbPathname, pers_lldb_purpose_e ePurpose, bo } } else + { bCanContinue = false; + } if (bCanContinue) { - int kissdb_state = 0; - size_t datasize = - (PersLldbPurpose_RCT == ePurpose) ? sizeof(PersistenceConfigurationKey_s) : sizeof(Data_LocalDB_s); - size_t keysize = - (PersLldbPurpose_RCT == ePurpose) ? PERS_RCT_MAX_LENGTH_RESOURCE_ID : PERS_DB_MAX_LENGTH_KEY_NAME; + size_t datasize = (PersLldbPurpose_RCT == ePurpose) ? sizeof(PersistenceConfigurationKey_s) : + PERS_DB_MAX_SIZE_KEY_DATA; + size_t keysize = (PersLldbPurpose_RCT == ePurpose) ? PERS_RCT_MAX_LENGTH_RESOURCE_ID : + PERS_DB_MAX_LENGTH_KEY_NAME; - if (bForceCreationIfNotPresent & (1 << 0) ) //check bit 0 - mode = KISSDB_OPEN_MODE_RWCREAT; + if (bForceCreationIfNotPresent & (1 << 0)) //check bit 0 0x0 (open) 0x1 (create) + { + openMode = KISSDB_OPEN_MODE_RWCREAT; //bit 0 is set + } + if(bForceCreationIfNotPresent & (1 << 1)) //check bit 1 + { + //bit 1 is set -> writeThrough mode 0x2 (open) 0x3 (create) + writeMode = KISSDB_WRITE_MODE_WT; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(LT_HDR), DLT_STRING(__FUNCTION__), DLT_STRING("Opening in write through mode:"), DLT_STRING("<"), + DLT_STRING(dbPathname), DLT_STRING(">, ")); + } + if( bForceCreationIfNotPresent & (1 << 2)) //check bit 2 + { + openMode = KISSDB_OPEN_MODE_RDONLY; //bit 2 is set 0x4 + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(LT_HDR), DLT_STRING(__FUNCTION__), DLT_STRING("Opening in read only mode:"), DLT_STRING("<"), + DLT_STRING(dbPathname), DLT_STRING(">, ")); + } -#ifdef __writeThrough - if(bForceCreationIfNotPresent & (1 << 1)) //check bit 1 - printf("cached \n"); + if (1 == checkIsLink(dbPathname, linkBuffer)) + { + path = linkBuffer; + } else - printf("uncached \n"); -#endif + { + path = dbPathname; + } + + //printKdb(&pLldbHandler->kissDb); - kissdb_state = KISSDB_open(&pLldbHandler->kissDb, dbPathname, mode, 256, keysize, datasize); - if (kissdb_state != 0) + if (pLldbHandler->kissDb.alreadyOpen == Kdb_false) //check if this instance has already opened the db before { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, - DLT_STRING("KISSDB_open: "); DLT_STRING("<<"); DLT_STRING(dbPathname); DLT_STRING(">>, "); DLT_STRING(" retval=<"); DLT_INT(kissdb_state); DLT_STRING(">"), DLT_STRING(strerror(errno))); + pLldbHandler->kissDb.semName = kdbGetShmName("-sem", path); + if (NULL == pLldbHandler->kissDb.semName) + { + return -1; + } + pLldbHandler->kissDb.kdbSem = sem_open(pLldbHandler->kissDb.semName, O_CREAT | O_EXCL, 0644, 1); + error = errno; //store errno -> (errno could be modified by following DLT_LOG) + if (pLldbHandler->kissDb.kdbSem == SEM_FAILED) //open failed + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(": first sem_open() with mode O_CREAT | O_EXCL failed with: "); DLT_STRING(strerror(error))); + if (error == EEXIST) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(": semaphore already exists: "); DLT_STRING(strerror(error))); + //try to open existing semaphore + pLldbHandler->kissDb.kdbSem = sem_open(pLldbHandler->kissDb.semName, O_CREAT, 0644, 0); + error = errno; + if (pLldbHandler->kissDb.kdbSem == SEM_FAILED) //open failed + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(": sem_open() for existing semaphore failed with error: "); DLT_STRING(strerror(error))); + return -1; + } + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":sem_open() failed:"); DLT_STRING(strerror(error))); + return -1; + } + } + } + if (-1 == sem_wait(pLldbHandler->kissDb.kdbSem)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": sem_wait() in open failed: "), + DLT_STRING(strerror(errno))); + } + kdbState = KISSDB_open(&pLldbHandler->kissDb, path, openMode, writeMode, HASHTABLE_SLOT_COUNT, keysize, datasize); + if (kdbState != 0) + { + if (kdbState == KISSDB_ERROR_WRONG_DATABASE_VERSION) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING("KISSDB_open: "); DLT_STRING("<"); DLT_STRING(path); DLT_STRING(">, "); DLT_STRING("database to be opened has wrong version! retval=<"); DLT_INT(kdbState); DLT_STRING(">")); + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING("KISSDB_open: "); DLT_STRING("<"); DLT_STRING(path); DLT_STRING(">, "); DLT_STRING(" retval=<"); DLT_INT(kdbState); DLT_STRING(">"), + DLT_STRING(strerror(errno))); + } bCanContinue = false; } - if (bCanContinue) + } + if (kdbState == 0) + { + pLldbHandler->kissDb.shared->refCount++; //increment reference to opened databases + if (-1 == sem_post(pLldbHandler->kissDb.kdbSem)) //release semaphore { - lldb_handles_InitHandle(pLldbHandler, ePurpose, dbPathname); - returnValue = pLldbHandler->dbHandler; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": End of open -> sem_post() failed: "),DLT_STRING(strerror(errno))); } - else + } + else + { + cleanKdbStruct(&pLldbHandler->kissDb); + + //in case of cleanup failure for semaphores, release semaphore + if (pLldbHandler->kissDb.kdbSem != NULL) { - /* clean up */ - returnValue = PERS_COM_FAILURE; - (void) lldb_handles_DeinitHandle(pLldbHandler->dbHandler); + if (-1 == sem_post(pLldbHandler->kissDb.kdbSem)) //release semaphore + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": End of open -> sem_post() in cleanup failed: "), + DLT_STRING(strerror(errno))); + } + } + + if(pLldbHandler->kissDb.semName != NULL) + { + free(pLldbHandler->kissDb.semName); + pLldbHandler->kissDb.semName = NULL; } } + if (bCanContinue) + { + lldb_handles_InitHandle(pLldbHandler, ePurpose, path); + returnValue = pLldbHandler->dbHandler; + } + else + { + /* clean up */ + returnValue = PERS_COM_FAILURE; + (void) lldb_handles_DeinitHandle(pLldbHandler->dbHandler); + } if (bLocked) + { (void) lldb_handles_Unlock(); - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING("End of open for:"); DLT_STRING("<<"); DLT_STRING(dbPathname); DLT_STRING(">>, "); ((PersLldbPurpose_RCT == ePurpose) ? DLT_STRING("RCT, ") : DLT_STRING("DB, ")); ((true == bForceCreationIfNotPresent) ? DLT_STRING("forced, ") : DLT_STRING("unforced, ")); DLT_STRING("retval=<"); DLT_INT(returnValue); DLT_STRING(">")); - - return returnValue; -} + } + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(LT_HDR), DLT_STRING(__FUNCTION__), DLT_STRING("End of open for:"), DLT_STRING("<"), + DLT_STRING(dbPathname), DLT_STRING(">, "), ((PersLldbPurpose_RCT == ePurpose) ? DLT_STRING("RCT, ") : DLT_STRING("DB, ")), + ((true == bForceCreationIfNotPresent) ? DLT_STRING("forced, ") : DLT_STRING("unforced, ")); DLT_STRING("retval=<"), DLT_INT(returnValue), + DLT_STRING(">")); + return returnValue; +} /** - * \close a key-value database + * \brief close a key-value database * \note : DB type is identified from dbPathname (based on extension) * * \param handlerDB [in] handler obtained with pers_lldb_open @@ -239,19 +371,24 @@ sint_t pers_lldb_open(str_t const * dbPathname, pers_lldb_purpose_e ePurpose, bo */ sint_t pers_lldb_close(sint_t handlerDB) { - int kissdb_state = 0; - sint_t returnValue = PERS_COM_SUCCESS; - lldb_handler_s* pLldbHandler = NIL; +#ifdef PFS_TEST + printf("START: pers_lldb_close for PID: %d \n", getpid()); +#endif + bool_t bLocked = false; + int kdbState = 0; + lldb_handler_s* pLldbHandler = NIL; + sint_t returnValue = PERS_COM_SUCCESS; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(handlerDB); DLT_STRING("...")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(handlerDB)); #ifdef __showTimeMeasurements long long duration = 0; long long KdbDuration = 0; long long writeDuration = 0; - struct timespec writeStart, writeEnd, kdbStart, kdbEnd, writebackStart, writebackEnd; + struct timespec writeStart, writeEnd, kdbStart, kdbEnd, writebackStart, + writebackEnd; duration = 0; KdbDuration = 0; writeDuration = 0; @@ -266,118 +403,168 @@ sint_t pers_lldb_close(sint_t handlerDB) pLldbHandler = lldb_handles_FindInUseHandle(handlerDB); if (NIL == pLldbHandler) + { returnValue = PERS_COM_FAILURE; + } } } else + { returnValue = PERS_COM_ERR_INVALID_PARAM; + } if (PERS_COM_SUCCESS == returnValue) { - //persist cached data to flash memory KISSDB* db = &pLldbHandler->kissDb; + if (-1 == sem_wait(db->kdbSem)) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(": sem_wait() in close failed: "), + DLT_STRING(strerror(errno))); + } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Closing database =<<"); DLT_STRING(pLldbHandler->dbPathname); DLT_STRING(">>, ")); - + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Closing database =<"); DLT_STRING(pLldbHandler->dbPathname); DLT_STRING(">, ")); - Kdb_wrlock(&db->shmem_info->cache_rwlock); + Kdb_wrlock(&db->shared->rwlock); //lock acces to shared status information - if (db->shmem_info->cache_initialised == Kdb_true) + if (db->shared->refCount > 0) + { + db->shared->refCount--; + } + if (db->shared->cacheCreated == Kdb_true) { - if (db->shmem_creator == Kdb_true) + if (db->shared->refCount == 0) { - //open existing cache in existing shared memory - if (db->shmem_cached_fd <= 0) + if (openCache(db) != 0) { - if (openCache(db) != 0) - { - Kdb_unlock(&db->shmem_info->cache_rwlock); - return PERS_COM_FAILURE; - } + Kdb_unlock(&db->shared->rwlock); + return PERS_COM_FAILURE; } #ifdef __showTimeMeasurements clock_gettime(CLOCK_ID, &writebackStart); #endif - if (pLldbHandler->ePurpose == PersLldbPurpose_DB) //write back to local database - writeBackKissDB(&pLldbHandler->kissDb, pLldbHandler); - else if (pLldbHandler->ePurpose == PersLldbPurpose_RCT) //write back to RCT database + + if (db->shared->openMode != KISSDB_OPEN_MODE_RDONLY) { - writeBackKissRCT(&pLldbHandler->kissDb, pLldbHandler); +#ifdef PFS_TEST + printf(" START: writeback of %d slots\n", pLldbHandler->kissDb.tbl->data->usedslots); +#endif + if (pLldbHandler->ePurpose == PersLldbPurpose_DB) //write back to local database + { + writeBackKissDB(&pLldbHandler->kissDb, pLldbHandler); + } + else + { + if (pLldbHandler->ePurpose == PersLldbPurpose_RCT) //write back to RCT database + { + writeBackKissRCT(&pLldbHandler->kissDb, pLldbHandler); + } + } +#ifdef PFS_TEST + printf(" END: writeback \n"); +#endif } + #ifdef __showTimeMeasurements clock_gettime(CLOCK_ID, &writebackEnd); #endif - if (db->shmem_info->cache_initialised) + //release reference object + db->tbl[0]->free(db->tbl[0]); + db->tbl[0] = NULL; + if (closeCache(db) != 0) { - db->tbl->free(db->tbl); - if (closeCache(db) != 0) - { - Kdb_unlock(&db->shmem_info->cache_rwlock); - return PERS_COM_FAILURE; - } + Kdb_unlock(&db->shared->rwlock); + return PERS_COM_FAILURE; + } + db->sharedCache = NULL; + if(db->sharedCacheFd) + { + close(db->sharedCacheFd); + db->sharedCacheFd = -1; + } + } + else //not the last instance, just unmap shared cache and free the name + { + //release reference object + db->tbl[0]->free(db->tbl[0]); + db->tbl[0] = NULL; + freeKdbShmemPtr(db->sharedCache, PERS_CACHE_MEMSIZE); + db->sharedCache = NULL; + if(db->sharedCacheFd) + { + close(db->sharedCacheFd); + db->sharedCacheFd = -1; } } } - Kdb_unlock(&db->shmem_info->cache_rwlock); + //no cache exists + Kdb_unlock(&db->shared->rwlock); #ifdef __showTimeMeasurements clock_gettime(CLOCK_ID, &kdbStart); #endif - kissdb_state = KISSDB_close(&pLldbHandler->kissDb); + kdbState = KISSDB_close(&pLldbHandler->kissDb); #ifdef __showTimeMeasurements clock_gettime(CLOCK_ID, &kdbEnd); #endif - if (kissdb_state == 0) + if (kdbState == 0) { if (!lldb_handles_DeinitHandle(pLldbHandler->dbHandler)) + { returnValue = PERS_COM_FAILURE; + } } else { - switch (kissdb_state) + switch (kdbState) { - case KISSDB_ERROR_UNMAP_SHM: + case KISSDB_ERROR_CLOSE_SHM: { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING("KISSDB_close: "); DLT_STRING("Could not unmap shared memory object, retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + DLT_STRING("KISSDB_close: "); DLT_STRING("Could not close shared memory object, retval=<"); DLT_INT(kdbState); DLT_STRING(">")); break; } - case KISSDB_ERROR_CLOSE_SHM: + default: { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING("KISSDB_close: "); DLT_STRING("Could not close shared memory object, retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + DLT_STRING("KISSDB_close: "); DLT_STRING("Could not close database, retval=<"); DLT_INT(kdbState); DLT_STRING(">")); break; } - default: - break; } returnValue = PERS_COM_FAILURE; } } + if (bLocked) + { (void) lldb_handles_Unlock(); + } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(handlerDB); DLT_STRING(" retval=<"); DLT_INT(returnValue); DLT_STRING(">")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(handlerDB); DLT_STRING(" retval=<"); DLT_INT(returnValue); DLT_STRING(">")); #ifdef __showTimeMeasurements clock_gettime(CLOCK_ID, &writeEnd); writeDuration += getNsDuration(&writebackStart, &writebackEnd); - printf("Writeback to flash duration for %s => %f ms\n", pLldbHandler->dbPathname, (double)((double)writeDuration/NANO2MIL)); + printf("Writeback to flash duration for %s => %f ms\n", + pLldbHandler->dbPathname, (double)((double)writeDuration/NANO2MIL)); KdbDuration += getNsDuration(&kdbStart, &kdbEnd); - printf("KISSDB_close duration for %s => %f ms\n", pLldbHandler->dbPathname, (double)((double)KdbDuration/NANO2MIL)); + printf("KISSDB_close duration for %s => %f ms\n", pLldbHandler->dbPathname, + (double)((double)KdbDuration/NANO2MIL)); duration += getNsDuration(&writeStart, &writeEnd); - printf("Overall Close duration for %s => %f ms\n", pLldbHandler->dbPathname, (double)((double)duration/NANO2MIL)); + printf("Overall Close duration for %s => %f ms\n", pLldbHandler->dbPathname, + (double)((double)duration/NANO2MIL)); #endif - return returnValue; -} - +#ifdef PFS_TEST + printf("END: pers_lldb_close for PID: %d \n", getpid()); +#endif + return returnValue; +} /** * \writeback cache of RCT key-value database @@ -385,48 +572,59 @@ sint_t pers_lldb_close(sint_t handlerDB) */ static sint_t writeBackKissRCT(KISSDB* db, lldb_handler_s* pLldbHandler) { - int kissdb_state = 0; + char* metaKey; + char* ptr; int idx = 0; - sint_t returnValue = PERS_COM_SUCCESS; - //lldb_handler_s* pLldbHandler = NIL; + int kdbState = 0; + int32_t bytesDeleted = 0; + int32_t bytesWritten = 0; pers_lldb_cache_flag_e eFlag; - char* ptr; qnobj_t obj; - char tmp_key[PERS_RCT_MAX_LENGTH_RESOURCE_ID]; + sint_t returnValue = PERS_COM_SUCCESS; + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("START writeback for RCT: "), + DLT_STRING(pLldbHandler->dbPathname)); - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("START writeback for RCT: "), DLT_STRING(db->shmem_ht_name) ); + setMemoryAddress(db->sharedCache, db->tbl[0]); - while (db->tbl->getnext(db->tbl, &obj, &idx) == true) + while (db->tbl[0]->getnext(db->tbl[0], &obj, &idx) == true) { ptr = obj.data; eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; ptr += 2 * (sizeof(int)); - (void) strncpy(tmp_key, obj.name, PERS_RCT_MAX_LENGTH_RESOURCE_ID); + metaKey = obj.name; //check how data should be persisted switch (eFlag) { case CachedDataDelete: //data must be deleted from file { - kissdb_state = KISSDB_delete(&pLldbHandler->kissDb, tmp_key); - if (kissdb_state != 0) + kdbState = KISSDB_delete(&pLldbHandler->kissDb, metaKey, &bytesDeleted); + if (kdbState != 0) { - if (kissdb_state == 1) + if (kdbState == 1) + { DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: RCT key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("not found in database file, retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: RCT key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("not found in database file, retval=<"); DLT_INT(kdbState); + DLT_STRING(">")); + } else + { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: RCT key=<");DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: RCT key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kdbState); DLT_STRING(">")); + } } break; } case CachedDataWrite: //data must be written to file { - kissdb_state = KISSDB_put(&pLldbHandler->kissDb, tmp_key, ptr); - if (kissdb_state != 0) + kdbState = KISSDB_put(&pLldbHandler->kissDb, metaKey, ptr, sizeof(PersistenceConfigurationKey_s), &bytesWritten); + if (kdbState != 0) + { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_put: RCT key=<");DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error: Writing back to file failed with retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_put: RCT key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("Writing back to file failed with retval=<"); + DLT_INT(kdbState); DLT_STRING(">")); + } break; } default: @@ -436,39 +634,40 @@ static sint_t writeBackKissRCT(KISSDB* db, lldb_handler_s* pLldbHandler) free(obj.data); } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("END writeback for RCT: "), DLT_STRING(db->shmem_ht_name) ); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("END writeback for RCT: "), + DLT_STRING(pLldbHandler->dbPathname)); return returnValue; } - - - /** - * \writeback cache of local DB key-value database - * \return 0 for success, negative value otherway (see pers_error_codes.h) + * Write back the data in cache to database file + * @param db + * @param pLldbHandler + * @return 0 for success, negative value otherway (see pers_error_codes.h) */ static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler) { - int kissdb_state = 0; + char* metaKey; + char* ptr; + Data_LocalDB_s insert = {{0},0}; + int datasize = 0; int idx = 0; - sint_t returnValue = PERS_COM_SUCCESS; - //lldb_handler_s* pLldbHandler = NIL; + int kdbState = 0; + int32_t bytesDeleted = 0; + int32_t bytesWritten = 0; pers_lldb_cache_flag_e eFlag; - char* ptr; qnobj_t obj; + sint_t returnValue = PERS_COM_SUCCESS; - char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; - Data_LocalDB_s insert = { { 0 } }; - int datasize = 0; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("START writeback for DB: "), + DLT_STRING(pLldbHandler->dbPathname)); - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("START writeback for DB: "), DLT_STRING(db->shmem_ht_name) ); + setMemoryAddress(db->sharedCache, db->tbl[0]); - while (db->tbl->getnext(db->tbl, &obj, &idx) == true) + while (db->tbl[0]->getnext(db->tbl[0], &obj, &idx) == true) { //get flag and datasize ptr = obj.data; @@ -476,7 +675,7 @@ static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler) ptr += sizeof(int); datasize = *(int*) ptr; //pointer in obj.data to datasize ptr += sizeof(int); //pointer in obj.data to data - (void) strncpy(tmp_key, obj.name, PERS_DB_MAX_LENGTH_KEY_NAME); + metaKey = obj.name; //check how data should be persisted switch (eFlag) @@ -484,15 +683,21 @@ static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler) case CachedDataDelete: //data must be deleted from file { //delete key-value pair from database file - kissdb_state = KISSDB_delete(&pLldbHandler->kissDb, tmp_key); - if (kissdb_state != 0) + kdbState = KISSDB_delete(&pLldbHandler->kissDb, metaKey, &bytesDeleted); + if (kdbState != 0) { - if (kissdb_state == 1) + if (kdbState == 1) + { DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("not found in database file, retval=<"); DLT_INT(kissdb_state); DLT_STRING(">")); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("not found in database file, retval=<"); DLT_INT(kdbState); + DLT_STRING(">")); + } else + { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<");DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kissdb_state); DLT_STRING(">"); DLT_STRING("Error Message: ");DLT_STRING(strerror(errno))); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kdbState); DLT_STRING(">"); + DLT_STRING("Error Message: "); DLT_STRING(strerror(errno))); + } } break; } @@ -500,11 +705,13 @@ static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler) { (void) memcpy(insert.m_data, ptr, datasize); insert.m_dataSize = datasize; - - kissdb_state = KISSDB_put(&pLldbHandler->kissDb, tmp_key, &insert); //store data followed by datasize - if (kissdb_state != 0) + kdbState = KISSDB_put(&pLldbHandler->kissDb, metaKey, &insert, insert.m_dataSize, &bytesWritten); //store data followed by datasize + if (kdbState != 0) + { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_put: key=<");DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error: Writing back to file failed with retval=<"); DLT_INT(kissdb_state); DLT_STRING(">"); DLT_STRING("Error Message: ");DLT_STRING(strerror(errno))); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_put: key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("Writing back to file failed with retval=<"); + DLT_INT(kdbState); DLT_STRING(">"); DLT_STRING("Error Message: "); DLT_STRING(strerror(errno))); + } break; } default: @@ -513,17 +720,12 @@ static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler) free(obj.name); free(obj.data); } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("END writeback for DB: "), DLT_STRING(db->shmem_ht_name) ); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("END writeback for DB: "), + DLT_STRING(pLldbHandler->dbPathname)); return returnValue; } - - - - - - /** * \brief write a key-value pair into database * \note : DB type is identified from dbPathname (based on extension) @@ -537,15 +739,10 @@ static sint_t writeBackKissDB(KISSDB* db, lldb_handler_s* pLldbHandler) * * \return 0 for success, negative value otherway (see pers_error_codes.h) */ -sint_t pers_lldb_write_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const * key, str_t const * data, - sint_t dataSize) +sint_t pers_lldb_write_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const* key, str_t const* data, sint_t dataSize) { sint_t eErrorCode = PERS_COM_SUCCESS; - //int i =0; - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Datatest="); DLT_RAW(data,dataSize); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>"); DLT_STRING(" Data Size=<<"); DLT_INT(dataSize); DLT_STRING(">>...")); - switch (ePurpose) { case PersLldbPurpose_DB: @@ -555,7 +752,7 @@ sint_t pers_lldb_write_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t } case PersLldbPurpose_RCT: { - eErrorCode = SetDataInKissRCT(handlerDB, key, (PersistenceConfigurationKey_s const *) data); + eErrorCode = SetDataInKissRCT(handlerDB, key, (PersistenceConfigurationKey_s const*) data); break; } default: @@ -579,8 +776,7 @@ sint_t pers_lldb_write_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t * * \return read size, or negative value in case of error (see pers_error_codes.h) */ -sint_t pers_lldb_read_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const * key, pstr_t dataBuffer_out, - sint_t bufSize) +sint_t pers_lldb_read_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const* key, pstr_t dataBuffer_out, sint_t bufSize) { sint_t eErrorCode = PERS_COM_SUCCESS; @@ -614,7 +810,7 @@ sint_t pers_lldb_read_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t * \param key [in] key's name * \return size of the value corresponding to the key, or negative value in case of error (see pers_error_codes.h) */ -sint_t pers_lldb_get_key_size(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const * key) +sint_t pers_lldb_get_key_size(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const* key) { sint_t eErrorCode = PERS_COM_SUCCESS; @@ -645,7 +841,7 @@ sint_t pers_lldb_get_key_size(sint_t handlerDB, pers_lldb_purpose_e ePurpose, st * * \return 0 for success, negative value otherway (see pers_error_codes.h) */ -sint_t pers_lldb_delete_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const * key) +sint_t pers_lldb_delete_key(sint_t handlerDB, pers_lldb_purpose_e ePurpose, str_t const* key) { sint_t eErrorCode = PERS_COM_SUCCESS; @@ -738,29 +934,19 @@ sint_t pers_lldb_get_keys_list(sint_t handlerDB, pers_lldb_purpose_e ePurpose, p break; } } - return eErrorCode; } -//TODO add write through compatibility static sint_t DeleteDataFromKissDB(sint_t dbHandler, pconststr_t key) { bool_t bCanContinue = true; - sint_t delete_size = PERS_COM_FAILURE; - lldb_handler_s* pLldbHandler = NIL; bool_t bLocked = false; - char m_data[sizeof(Data_LocalDB_s)] = {0}; - pers_lldb_cache_flag_e eFlag; - void *val; - char *ptr; - int status = PERS_COM_FAILURE; - int datasize = 0; - Kdb_bool not_found = Kdb_false; - size_t size = 0; - Data_Cached_s data_cached = { 0 }; + int kdbState = 0; + lldb_handler_s* pLldbHandler = NIL; + sint_t bytesDeleted = PERS_COM_FAILURE; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>...")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">")); if ((dbHandler >= 0) && (NIL != key)) { @@ -769,135 +955,69 @@ static sint_t DeleteDataFromKissDB(sint_t dbHandler, pconststr_t key) bLocked = true; pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); if (NIL == pLldbHandler) + { bCanContinue = false; + } } } else + { bCanContinue = false; + } + if (bCanContinue) { - KISSDB* db = &pLldbHandler->kissDb; - - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(db->shmem_ht_name) ); - - Kdb_wrlock(&db->shmem_info->cache_rwlock); - - char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; - (void) strncpy(tmp_key, key, PERS_DB_MAX_LENGTH_KEY_NAME); - data_cached.eFlag = CachedDataDelete; - data_cached.m_dataSize = 0; - - //if cache not already created - if (db->shmem_info->cache_initialised == Kdb_false) - { - if (createCache(db) != 0) - { - Kdb_unlock(&db->shmem_info->cache_rwlock); - return PERS_COM_FAILURE; - } - } - else //open cache + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); + if ( KISSDB_WRITE_MODE_WC == pLldbHandler->kissDb.shared->writeMode) { - if (openCache(db) != 0) - { - Kdb_unlock(&db->shmem_info->cache_rwlock); - return PERS_COM_FAILURE; - } + bytesDeleted = deleteFromCache(&pLldbHandler->kissDb, (char*) key); } - val = db->tbl->get(db->tbl, tmp_key, &size); - if (NULL != val) //check if key to be deleted is in Cache + else //write through { - ptr = val; - eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; - ptr += sizeof(int); - datasize = *(int*) ptr; - //Mark data in cache as deleted - if (eFlag != CachedDataDelete) - { - if (db->tbl->put(db->tbl, tmp_key, &data_cached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) //do not store any data - { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Failed to mark data in cache as deleted")); - delete_size = PERS_COM_ERR_NOT_FOUND; - not_found = Kdb_true; - } - else - delete_size = datasize; - } - } - else //check if key to be deleted is in database file - { - //get dataSize - status = KISSDB_get(&pLldbHandler->kissDb, tmp_key, m_data); - if (status == 0) - { - ptr = m_data; - ptr += PERS_DB_MAX_SIZE_KEY_DATA; - datasize = *(int*) ptr; - //put information about the key to be deleted in cache (deletion in file happens at system shutdown) - if (db->tbl->put(db->tbl, tmp_key, &data_cached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) - { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Failed to mark existing data as deleted")); - delete_size = PERS_COM_ERR_NOT_FOUND; - } - else - delete_size = datasize; - } - else + kdbState = KISSDB_delete(&pLldbHandler->kissDb, key, &bytesDeleted); + if (kdbState != 0) { - if (status == 1) + if (kdbState == 1) { - not_found = Kdb_true; DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("not found, retval=<"); DLT_INT(status); DLT_STRING(">")); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("not found in database file, retval=<"); DLT_INT(kdbState); + DLT_STRING(">")); } else { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(status); DLT_STRING(">")); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kdbState); DLT_STRING(">"); + DLT_STRING("Error Message: "); DLT_STRING(strerror(errno))); } } } - - if (not_found == Kdb_true) //key not found, - delete_size = PERS_COM_ERR_NOT_FOUND; - Kdb_unlock(&db->shmem_info->cache_rwlock); + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); } if (bLocked) + { (void) lldb_handles_Unlock(); + } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(delete_size); DLT_STRING(">")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("retval=<"); + DLT_INT(bytesDeleted); DLT_STRING(">")); - return delete_size; + return bytesDeleted; } - - -//TODO add write through compatibility static sint_t DeleteDataFromKissRCT(sint_t dbHandler, pconststr_t key) { bool_t bCanContinue = true; - sint_t delete_size = PERS_COM_FAILURE; - lldb_handler_s* pLldbHandler = NIL; bool_t bLocked = false; - char m_data[sizeof(Data_LocalDB_s)] = {0}; - pers_lldb_cache_flag_e eFlag; - void *val; - char *ptr; - int status = PERS_COM_FAILURE; - int datasize = 0; - Kdb_bool not_found = Kdb_false; - size_t size = 0; - Data_Cached_s data_cached = { 0 }; + int kdbState = 0; + lldb_handler_s* pLldbHandler = NIL; + sint_t bytesDeleted = PERS_COM_FAILURE; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>...")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">")); if ((dbHandler >= 0) && (NIL != key)) { @@ -906,123 +1026,67 @@ static sint_t DeleteDataFromKissRCT(sint_t dbHandler, pconststr_t key) bLocked = true; pLldbHandler = lldb_handles_FindInUseHandle(dbHandler); if (NIL == pLldbHandler) + { bCanContinue = false; + } } } else + { bCanContinue = false; + } if (bCanContinue) { - KISSDB* db = &pLldbHandler->kissDb; - - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(db->shmem_ht_name) ); - - Kdb_wrlock(&db->shmem_info->cache_rwlock); - - char tmp_key[PERS_RCT_MAX_LENGTH_RESOURCE_ID]; - (void) strncpy(tmp_key, key, PERS_RCT_MAX_LENGTH_RESOURCE_ID); - data_cached.eFlag = CachedDataDelete; - data_cached.m_dataSize = 0; - - //if cache not already created - if (db->shmem_info->cache_initialised == Kdb_false) + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); + if ( KISSDB_WRITE_MODE_WC == pLldbHandler->kissDb.shared->writeMode) { - if (createCache(db) != 0) - { - Kdb_unlock(&db->shmem_info->cache_rwlock); - return PERS_COM_FAILURE; - } + bytesDeleted = deleteFromCache(&pLldbHandler->kissDb, (char*) key); } - else //open cache + else //write through { - if (openCache(db) != 0) + kdbState = KISSDB_delete(&pLldbHandler->kissDb, key, &bytesDeleted); + if (kdbState != 0) { - Kdb_unlock(&db->shmem_info->cache_rwlock); - return PERS_COM_FAILURE; - } - } - //get dataSize - val = db->tbl->get(db->tbl, tmp_key, &size); - if (NULL != val) //check if key to be deleted is in Cache - { - ptr = val; - eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; - ptr += sizeof(int); - datasize = *(int*) ptr; - - //Mark data in cache as deleted - if (eFlag != CachedDataDelete) - { - if (db->tbl->put(db->tbl, tmp_key, &data_cached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) + if (kdbState == 1) { - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Failed to mark RCT data in cache as deleted")); - delete_size = PERS_COM_ERR_NOT_FOUND; - not_found = Kdb_true; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("not found in database file, retval=<"); DLT_INT(kdbState); + DLT_STRING(">")); } else - delete_size = datasize; - } - } - else //check if key to be deleted is in database file - { - status = KISSDB_get(&pLldbHandler->kissDb, tmp_key, m_data); - if (status == 0) - { - //Data to be deleted is not in cache, but was found in local database - //put information about the key to be deleted in cache (deletion in file happens at system shutdown) - if (db->tbl->put(db->tbl, tmp_key, &data_cached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Failed to mark existing RCT data as deleted")); - delete_size = PERS_COM_ERR_NOT_FOUND; - } - else - delete_size = sizeof(PersistenceConfigurationKey_s); - } - else - { - if (status == 1) - { - not_found = Kdb_true; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("not found, retval=<"); DLT_INT(status); DLT_STRING(">")); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_delete: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kdbState); DLT_STRING(">"); + DLT_STRING("Error Message: "); DLT_STRING(strerror(errno))); } - else - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(status); DLT_STRING(">")); } } - if (not_found == Kdb_true) - delete_size = PERS_COM_ERR_NOT_FOUND; - - Kdb_unlock(&db->shmem_info->cache_rwlock); + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); } if (bLocked) + { (void) lldb_handles_Unlock(); + } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(delete_size); DLT_STRING(">")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("handlerDB="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("retval=<"); + DLT_INT(bytesDeleted); DLT_STRING(">")); - return delete_size; + return bytesDeleted; } - - -//TODO add write through compatibility static sint_t GetAllKeysFromKissLocalDB(sint_t dbHandler, pstr_t buffer, sint_t size) { bool_t bCanContinue = true; - sint_t result = 0; + bool_t bLocked = false; bool_t bOnlySizeNeeded = (NIL == buffer); lldb_handler_s* pLldbHandler = NIL; - bool_t bLocked = false; + sint_t result = 0; - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - //DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("buffer="); DLT_UINT((uint_t)buffer); DLT_STRING("size="); DLT_INT(size); DLT_STRING("...")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("buffer="); DLT_UINT((uint_t)buffer); DLT_STRING("size="); DLT_INT(size)); if (dbHandler >= 0) { @@ -1038,7 +1102,8 @@ static sint_t GetAllKeysFromKissLocalDB(sint_t dbHandler, pstr_t buffer, sint_t else { if (PersLldbPurpose_DB != pLldbHandler->ePurpose) - {/* this would be very bad */ + { + /* this would be very bad */ bCanContinue = false; result = PERS_COM_FAILURE; } @@ -1055,34 +1120,38 @@ static sint_t GetAllKeysFromKissLocalDB(sint_t dbHandler, pstr_t buffer, sint_t if (bCanContinue) { if ((buffer != NIL) && (size > 0)) + { (void) memset(buffer, 0, (size_t) size); + } - Kdb_wrlock(&pLldbHandler->kissDb.shmem_info->cache_rwlock); + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); result = getListandSize(&pLldbHandler->kissDb, buffer, size, bOnlySizeNeeded, PersLldbPurpose_DB); - Kdb_unlock(&pLldbHandler->kissDb.shmem_info->cache_rwlock); + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); if (result < 0) + { result = PERS_COM_FAILURE; + } } if (bLocked) + { (void) lldb_handles_Unlock(); + } - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - //DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("retval=<"); DLT_INT(result); DLT_STRING(">")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("retval=<"); DLT_INT(result); DLT_STRING(">")); return result; } - -//TODO add write through compatibility static sint_t GetAllKeysFromKissRCT(sint_t dbHandler, pstr_t buffer, sint_t size) { bool_t bCanContinue = true; - sint_t result = 0; + bool_t bLocked = false; bool_t bOnlySizeNeeded = (NIL == buffer); lldb_handler_s* pLldbHandler = NIL; - bool_t bLocked = false; + sint_t result = 0; - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - //DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("buffer="); DLT_UINT((uint_t)buffer); DLT_STRING("size="); DLT_INT(size); DLT_STRING("...")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("buffer="); DLT_UINT((uint_t)buffer); DLT_STRING("size="); DLT_INT(size)); if (dbHandler >= 0) { @@ -1098,7 +1167,8 @@ static sint_t GetAllKeysFromKissRCT(sint_t dbHandler, pstr_t buffer, sint_t size else { if (PersLldbPurpose_RCT != pLldbHandler->ePurpose) - {/* this would be very bad */ + { + /* this would be very bad */ bCanContinue = false; result = PERS_COM_FAILURE; } @@ -1115,34 +1185,41 @@ static sint_t GetAllKeysFromKissRCT(sint_t dbHandler, pstr_t buffer, sint_t size if (bCanContinue) { if ((buffer != NIL) && (size > 0)) + { (void) memset(buffer, 0, (size_t) size); - Kdb_wrlock(&pLldbHandler->kissDb.shmem_info->cache_rwlock); + } + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); result = getListandSize(&pLldbHandler->kissDb, buffer, size, bOnlySizeNeeded, PersLldbPurpose_RCT); - Kdb_unlock(&pLldbHandler->kissDb.shmem_info->cache_rwlock); + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); if (result < 0) + { result = PERS_COM_FAILURE; + } } if (bLocked) + { (void) lldb_handles_Unlock(); + } - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - //DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("retval=<"); DLT_INT(result); DLT_STRING(">")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("retval=<"); DLT_INT(result); DLT_STRING(">")); return result; } - -//TODO add write through compatibility static sint_t SetDataInKissLocalDB(sint_t dbHandler, pconststr_t key, pconststr_t data, sint_t dataSize) { bool_t bCanContinue = true; - sint_t size_written = PERS_COM_FAILURE; - lldb_handler_s* pLldbHandler = NIL; bool_t bLocked = false; - Data_Cached_s data_cached = { 0 }; + Data_Cached_s dataCached = { 0 }; + int kdbState = 0; + lldb_handler_s* pLldbHandler = NIL; + sint_t bytesWritten = PERS_COM_FAILURE; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("size<<"); DLT_INT(dataSize); DLT_STRING(">> ...")); + + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("size<"); + DLT_INT(dataSize); DLT_STRING(">")); if ((dbHandler >= 0) && (NIL != key) && (NIL != data) && (dataSize > 0)) { @@ -1153,14 +1230,15 @@ static sint_t SetDataInKissLocalDB(sint_t dbHandler, pconststr_t key, pconststr_ if (NIL == pLldbHandler) { bCanContinue = false; - size_written = PERS_COM_ERR_INVALID_PARAM; + bytesWritten = PERS_COM_ERR_INVALID_PARAM; } else { if (PersLldbPurpose_DB != pLldbHandler->ePurpose) - {/* this would be very bad */ + { + /* this would be very bad */ bCanContinue = false; - size_written = PERS_COM_FAILURE; + bytesWritten = PERS_COM_FAILURE; } } } @@ -1168,43 +1246,59 @@ static sint_t SetDataInKissLocalDB(sint_t dbHandler, pconststr_t key, pconststr_ else { bCanContinue = false; - size_written = PERS_COM_ERR_INVALID_PARAM; + bytesWritten = PERS_COM_ERR_INVALID_PARAM; } if (bCanContinue) { - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(pLldbHandler->dbPathname) ); + char* metaKey = (char*) key; + dataCached.eFlag = CachedDataWrite; + dataCached.m_dataSize = dataSize; + (void) memcpy(dataCached.m_data, data, (size_t) dataSize); - //TODO add write through (call KissDB_put) - char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; - (void) strncpy(tmp_key, key, PERS_DB_MAX_LENGTH_KEY_NAME); - data_cached.eFlag = CachedDataWrite; - data_cached.m_dataSize = dataSize; - (void) memcpy(data_cached.m_data, data, (size_t) dataSize); - size_written = putToCache(&pLldbHandler->kissDb, dataSize, (char*) &tmp_key, &data_cached); + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); + if ( KISSDB_WRITE_MODE_WC == pLldbHandler->kissDb.shared->writeMode) + { + bytesWritten = putToCache(&pLldbHandler->kissDb, dataSize, (char*) metaKey, &dataCached); + } + else + { + if (KISSDB_OPEN_MODE_RDONLY != pLldbHandler->kissDb.shared->openMode) + { + kdbState = KISSDB_put(&pLldbHandler->kissDb, metaKey, dataCached.m_data, dataCached.m_dataSize, &bytesWritten); + if (kdbState != 0) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_put: key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("WriteThrough to file failed with retval=<"); DLT_INT(bytesWritten); DLT_STRING(">")); + } + } + } + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); } if (bLocked) + { (void) lldb_handles_Unlock(); + } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("size<<"); DLT_INT(dataSize); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_written); DLT_STRING(">")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("size<"); + DLT_INT(dataSize); DLT_STRING(">, "); DLT_STRING("retval=<"); DLT_INT(bytesWritten); DLT_STRING(">")); - return size_written; + return bytesWritten; } -//TODO add write through compatibility -static sint_t SetDataInKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s const * pConfig) +static sint_t SetDataInKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s const* pConfig) { bool_t bCanContinue = true; - sint_t size_written = PERS_COM_FAILURE; - lldb_handler_s* pLldbHandler = NIL; bool_t bLocked = false; - Data_Cached_RCT_s data_cached = { 0 }; + Data_Cached_RCT_s dataCached = { 0 }; + int kdbState = 0; + lldb_handler_s* pLldbHandler = NIL; + sint_t bytesWritten = PERS_COM_FAILURE; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>...")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">")); if ((dbHandler >= 0) && (NIL != key) && (NIL != pConfig)) { @@ -1215,14 +1309,15 @@ static sint_t SetDataInKissRCT(sint_t dbHandler, pconststr_t key, PersistenceCon if (NIL == pLldbHandler) { bCanContinue = false; - size_written = PERS_COM_ERR_INVALID_PARAM; + bytesWritten = PERS_COM_ERR_INVALID_PARAM; } else { if (PersLldbPurpose_RCT != pLldbHandler->ePurpose) - {/* this would be very bad */ + { + /* this would be very bad */ bCanContinue = false; - size_written = PERS_COM_FAILURE; + bytesWritten = PERS_COM_FAILURE; } /* to not use DLT while mutex locked */ } @@ -1231,42 +1326,59 @@ static sint_t SetDataInKissRCT(sint_t dbHandler, pconststr_t key, PersistenceCon else { bCanContinue = false; - size_written = PERS_COM_ERR_INVALID_PARAM; + bytesWritten = PERS_COM_ERR_INVALID_PARAM; } if (bCanContinue) { - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(pLldbHandler->dbPathname) ); - - //TODO add RCT write through (call KissDB_put) int dataSize = sizeof(PersistenceConfigurationKey_s); - char tmp_key[PERS_RCT_MAX_LENGTH_RESOURCE_ID]; - (void) strncpy(tmp_key, key, PERS_RCT_MAX_LENGTH_RESOURCE_ID); - data_cached.eFlag = CachedDataWrite; - data_cached.m_dataSize = dataSize; - (void) memcpy(data_cached.m_data, pConfig, (size_t) dataSize); - size_written = putToCache(&pLldbHandler->kissDb, dataSize, (char*) &tmp_key, &data_cached); + char* metaKey = (char*) key; + dataCached.eFlag = CachedDataWrite; + dataCached.m_dataSize = dataSize; + (void) memcpy(dataCached.m_data, pConfig, (size_t) dataSize); + + + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); + if ( KISSDB_WRITE_MODE_WC == pLldbHandler->kissDb.shared->writeMode) + { + bytesWritten = putToCache(&pLldbHandler->kissDb, dataSize, (char*) metaKey, &dataCached); + } + else + { + + if (KISSDB_OPEN_MODE_RDONLY != pLldbHandler->kissDb.shared->openMode) + { + kdbState = KISSDB_put(&pLldbHandler->kissDb, metaKey, dataCached.m_data, dataCached.m_dataSize, &bytesWritten); + if (kdbState != 0) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_put: RCT key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("WriteThrough to file failed with retval=<"); DLT_INT(bytesWritten); DLT_STRING(">")); + } + } + } + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); + } if (bLocked) + { (void) lldb_handles_Unlock(); + } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_written); DLT_STRING(">")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("retval=<"); + DLT_INT(bytesWritten); DLT_STRING(">")); - return size_written; + return bytesWritten; } - -//TODO add write through compatibility static sint_t GetKeySizeFromKissLocalDB(sint_t dbHandler, pconststr_t key) { bool_t bCanContinue = true; - sint_t size_read = PERS_COM_FAILURE; - lldb_handler_s* pLldbHandler = NIL; bool_t bLocked = false; + lldb_handler_s* pLldbHandler = NIL; + sint_t bytesRead = PERS_COM_FAILURE; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">> ...")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">")); if ((dbHandler >= 0) && (NIL != key)) { @@ -1277,14 +1389,15 @@ static sint_t GetKeySizeFromKissLocalDB(sint_t dbHandler, pconststr_t key) if (NIL == pLldbHandler) { bCanContinue = false; - size_read = PERS_COM_ERR_INVALID_PARAM; + bytesRead = PERS_COM_ERR_INVALID_PARAM; } else { if (PersLldbPurpose_DB != pLldbHandler->ePurpose) - {/* this would be very bad */ + { + /* this would be very bad */ bCanContinue = false; - size_read = PERS_COM_FAILURE; + bytesRead = PERS_COM_FAILURE; } } } @@ -1292,38 +1405,47 @@ static sint_t GetKeySizeFromKissLocalDB(sint_t dbHandler, pconststr_t key) else { bCanContinue = false; - size_read = PERS_COM_ERR_INVALID_PARAM; + bytesRead = PERS_COM_ERR_INVALID_PARAM; } if (bCanContinue) { - char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; - (void) strncpy(tmp_key, key, PERS_DB_MAX_LENGTH_KEY_NAME); - size_read = getFromCache(&pLldbHandler->kissDb, &tmp_key, NULL, 0, true); - if (size_read == PERS_STATUS_KEY_NOT_IN_CACHE) - size_read = getFromDatabaseFile(&pLldbHandler->kissDb, &tmp_key, NULL, PersLldbPurpose_DB, 0, true); + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); + if ( KISSDB_WRITE_MODE_WC == pLldbHandler->kissDb.shared->writeMode) + { + bytesRead = getFromCache(&pLldbHandler->kissDb, (char*) key, NULL, 0, true); + if (bytesRead == PERS_STATUS_KEY_NOT_IN_CACHE) + { + bytesRead = getFromDatabaseFile(&pLldbHandler->kissDb, (char*) key, NULL, 0); + } + } + else + { + bytesRead = getFromDatabaseFile(&pLldbHandler->kissDb, (char*) key, NULL, 0); + } + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); } if (bLocked) + { (void) lldb_handles_Unlock(); - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_read); DLT_STRING(">")); + } + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("retval=<"); + DLT_INT(bytesRead); DLT_STRING(">")); - return size_read; + return bytesRead; } - - - -//TODO add write through compatibility /* return no of bytes read, or negative value in case of error */ static sint_t GetDataFromKissLocalDB(sint_t dbHandler, pconststr_t key, pstr_t buffer_out, sint_t bufSize) { bool_t bCanContinue = true; - sint_t size_read = PERS_COM_FAILURE; - lldb_handler_s* pLldbHandler = NIL; bool_t bLocked = false; + lldb_handler_s* pLldbHandler = NIL; + sint_t bytesRead = PERS_COM_FAILURE; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("bufsize=<<"); DLT_INT(bufSize); DLT_STRING(">> ... ")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("bufsize=<"); + DLT_INT(bufSize); DLT_STRING(">")); if ((dbHandler >= 0) && (NIL != key) && (NIL != buffer_out) && (bufSize > 0)) { @@ -1334,14 +1456,15 @@ static sint_t GetDataFromKissLocalDB(sint_t dbHandler, pconststr_t key, pstr_t b if (NIL == pLldbHandler) { bCanContinue = false; - size_read = PERS_COM_ERR_INVALID_PARAM; + bytesRead = PERS_COM_ERR_INVALID_PARAM; } else { if (PersLldbPurpose_DB != pLldbHandler->ePurpose) - {/* this would be very bad */ + { + /* this would be very bad */ bCanContinue = false; - size_read = PERS_COM_FAILURE; + bytesRead = PERS_COM_FAILURE; } /* to not use DLT while mutex locked */ } @@ -1350,42 +1473,47 @@ static sint_t GetDataFromKissLocalDB(sint_t dbHandler, pconststr_t key, pstr_t b else { bCanContinue = false; - size_read = PERS_COM_ERR_INVALID_PARAM; + bytesRead = PERS_COM_ERR_INVALID_PARAM; } if (bCanContinue) { - - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(pLldbHandler->dbPathname) ); - - char tmp_key[PERS_DB_MAX_LENGTH_KEY_NAME]; - (void) strncpy(tmp_key, key, PERS_DB_MAX_LENGTH_KEY_NAME); - size_read = getFromCache(&pLldbHandler->kissDb, &tmp_key, buffer_out, bufSize, false); - //if key is not already in cache - if (size_read == PERS_STATUS_KEY_NOT_IN_CACHE) - size_read = getFromDatabaseFile(&pLldbHandler->kissDb, &tmp_key, buffer_out, PersLldbPurpose_DB, bufSize, - false); + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); + if ( KISSDB_WRITE_MODE_WC == pLldbHandler->kissDb.shared->writeMode) + { + bytesRead = getFromCache(&pLldbHandler->kissDb, (char*) key, buffer_out, bufSize, false); + //if key is not already in cache + if (bytesRead == PERS_STATUS_KEY_NOT_IN_CACHE) + { + bytesRead = getFromDatabaseFile(&pLldbHandler->kissDb, (char*) key, buffer_out, bufSize); + } + } + else //write through mode -> only read from file + { + bytesRead = getFromDatabaseFile(&pLldbHandler->kissDb, (char*) key, buffer_out, bufSize); + } + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); } if (bLocked) + { (void) lldb_handles_Unlock(); + } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("bufsize=<<"); DLT_INT(bufSize); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_read); DLT_STRING(">")); - return size_read; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("bufsize=<"); + DLT_INT(bufSize); DLT_STRING(">, "); DLT_STRING("retval=<"); DLT_INT(bytesRead); DLT_STRING(">")); + return bytesRead; } - -//TODO add write through compatibility static sint_t GetDataFromKissRCT(sint_t dbHandler, pconststr_t key, PersistenceConfigurationKey_s* pConfig) { bool_t bCanContinue = true; - sint_t size_read = PERS_COM_FAILURE; - lldb_handler_s* pLldbHandler = NIL; bool_t bLocked = false; + lldb_handler_s* pLldbHandler = NIL; + sint_t bytesRead = PERS_COM_FAILURE; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">> ...")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">")); if ((dbHandler >= 0) && (NIL != key) && (NIL != pConfig)) { @@ -1396,14 +1524,15 @@ static sint_t GetDataFromKissRCT(sint_t dbHandler, pconststr_t key, PersistenceC if (NIL == pLldbHandler) { bCanContinue = false; - size_read = PERS_COM_ERR_INVALID_PARAM; + bytesRead = PERS_COM_ERR_INVALID_PARAM; } else { if (PersLldbPurpose_RCT != pLldbHandler->ePurpose) - {/* this would be very bad */ + { + /* this would be very bad */ bCanContinue = false; - size_read = PERS_COM_FAILURE; + bytesRead = PERS_COM_FAILURE; } /* to not use DLT while mutex locked */ } @@ -1412,34 +1541,39 @@ static sint_t GetDataFromKissRCT(sint_t dbHandler, pconststr_t key, PersistenceC else { bCanContinue = false; - size_read = PERS_COM_ERR_INVALID_PARAM; + bytesRead = PERS_COM_ERR_INVALID_PARAM; } //read RCT if (bCanContinue) { - //DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - // DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Working on DB: "), DLT_STRING(pLldbHandler->dbPathname) ); - - char tmp_key[PERS_RCT_MAX_LENGTH_RESOURCE_ID]; - (void) strncpy(tmp_key, key, PERS_RCT_MAX_LENGTH_RESOURCE_ID); - - size_read = getFromCache(&pLldbHandler->kissDb, &tmp_key, pConfig, sizeof(PersistenceConfigurationKey_s), false); - if (size_read == PERS_STATUS_KEY_NOT_IN_CACHE) - size_read = getFromDatabaseFile(&pLldbHandler->kissDb, &tmp_key, pConfig, PersLldbPurpose_RCT, - sizeof(PersistenceConfigurationKey_s), false); + Kdb_wrlock(&pLldbHandler->kissDb.shared->rwlock); + if ( KISSDB_WRITE_MODE_WC == pLldbHandler->kissDb.shared->writeMode) + { + bytesRead = getFromCache(&pLldbHandler->kissDb, (char*) key, pConfig, sizeof(PersistenceConfigurationKey_s), false); + if (bytesRead == PERS_STATUS_KEY_NOT_IN_CACHE) + { + bytesRead = getFromDatabaseFile(&pLldbHandler->kissDb, (char*) key, pConfig, sizeof(PersistenceConfigurationKey_s)); + } + } + else + { + bytesRead = getFromDatabaseFile(&pLldbHandler->kissDb, (char*) key, pConfig, sizeof(PersistenceConfigurationKey_s)); + } + Kdb_unlock(&pLldbHandler->kissDb.shared->rwlock); } if (bLocked) + { (void) lldb_handles_Unlock(); + } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<<"); DLT_STRING(key); DLT_STRING(">>, "); DLT_STRING("retval=<"); DLT_INT(size_read); DLT_STRING(">")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler="); DLT_INT(dbHandler); DLT_STRING("key=<"); DLT_STRING(key); DLT_STRING(">, "); DLT_STRING("retval=<"); + DLT_INT(bytesRead); DLT_STRING(">")); - return size_read; + return bytesRead; } - - static bool_t lldb_handles_Lock(void) { bool_t bEverythingOK = true; @@ -1448,7 +1582,7 @@ static bool_t lldb_handles_Lock(void) { bEverythingOK = false; DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("pthread_mutex_lock failed with error=<"); DLT_INT(siErr); DLT_STRING(">")); + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("pthread_mutex_lock failed with error=<"); DLT_INT(siErr); DLT_STRING(">")); } return bEverythingOK; } @@ -1462,7 +1596,7 @@ static bool_t lldb_handles_Unlock(void) { bEverythingOK = false; DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("pthread_mutex_unlock failed with error=<"); DLT_INT(siErr); DLT_STRING(">")); + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("pthread_mutex_unlock failed with error=<"); DLT_INT(siErr); DLT_STRING(">")); } return bEverythingOK; } @@ -1493,8 +1627,9 @@ static lldb_handler_s* lldb_handles_FindInUseHandle(sint_t dbHandler) } } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING((NIL!=pHandler) ? "Found handler <" : "ERROR can't find handler <"); DLT_INT(dbHandler); DLT_STRING(">"); DLT_STRING((NIL!=pHandler) ? (dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "in static area" : "in dynamic list") : "")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING((NIL!=pHandler) ? "Found handler <" : "ERROR can't find handler <"); DLT_INT(dbHandler); DLT_STRING(">"); + DLT_STRING((NIL!=pHandler) ? (dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "in static area" : "in dynamic list") : "")); return pHandler; } @@ -1511,6 +1646,7 @@ static lldb_handler_s* lldb_handles_FindAvailableHandle(void) { if (!g_sHandlers.asStaticHandles[siIndex].bIsAssigned) { + //INITIALIZE KISSDB struct /* index setting should be done only once at the initialization of the static array * Anyway, doing it here is more consistent */ g_sHandlers.asStaticHandles[siIndex].dbHandler = siIndex; @@ -1529,8 +1665,7 @@ static lldb_handler_s* lldb_handles_FindAvailableHandle(void) if (NIL == psListElemNew) { bCanContinue = false; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("malloc failed")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("malloc failed")); } else { @@ -1597,14 +1732,15 @@ static lldb_handler_s* lldb_handles_FindAvailableHandle(void) } } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING((NIL!=pHandler) ? "Found availble handler <" : "ERROR can't find available handler <"); DLT_INT((NIL!=pHandler) ? pHandler->dbHandler : (-1)); DLT_STRING(">"); DLT_STRING((NIL!=pHandler) ? (pHandler->dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "in static area" : "in dynamic list") : "")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING((NIL!=pHandler) ? "Found availble handler <" : "ERROR can't find available handler <"); + DLT_INT((NIL!=pHandler) ? pHandler->dbHandler : (-1)); DLT_STRING(">"); + DLT_STRING((NIL!=pHandler) ? (pHandler->dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "in static area" : "in dynamic list") : "")); return pHandler; } -static void lldb_handles_InitHandle(lldb_handler_s* psHandle_inout, pers_lldb_purpose_e ePurpose, - str_t const * dbPathname) +static void lldb_handles_InitHandle(lldb_handler_s* psHandle_inout, pers_lldb_purpose_e ePurpose, str_t const* dbPathname) { psHandle_inout->bIsAssigned = true; psHandle_inout->ePurpose = ePurpose; @@ -1669,43 +1805,40 @@ static bool_t lldb_handles_DeinitHandle(sint_t dbHandler) } } - DLT_LOG(persComLldbDLTCtx, DLT_LOG_INFO, - DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler=<"); DLT_INT(dbHandler); DLT_STRING("> "); DLT_STRING(bEverythingOK ? (dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "deinit handler in static area" : "deinit handler in dynamic list") : "ERROR - handler not found")); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(LT_HDR); DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("dbHandler=<"); DLT_INT(dbHandler); DLT_STRING("> "); + DLT_STRING(bEverythingOK ? (dbHandler <= PERS_LLDB_MAX_STATIC_HANDLES ? "deinit handler in static area" : "deinit handler in dynamic list") : "ERROR - handler not found")); return bEverythingOK; } - - -sint_t getFromCache(KISSDB* db, void* tmp_key, void* readBuffer, sint_t bufsize, bool_t sizeOnly) +sint_t getFromCache(KISSDB* db, void* metaKey, void* readBuffer, sint_t bufsize, bool_t sizeOnly) { - Kdb_bool cache_empty, key_deleted, key_not_found; - key_deleted = cache_empty = key_not_found = Kdb_false; - sint_t size_read = 0; - size_t size = 0; - int datasize = 0; char* ptr; + int datasize = 0; + Kdb_bool cacheEmpty, keyDeleted, keyNotFound; pers_lldb_cache_flag_e eFlag; + sint_t bytesRead = 0; + size_t size = 0; + void* val; - //if cache not already created - Kdb_wrlock(&db->shmem_info->cache_rwlock); + keyDeleted = cacheEmpty = keyNotFound = Kdb_false; - if (db->shmem_info->cache_initialised == Kdb_true) + //if cache already created + if (db->shared->cacheCreated == Kdb_true) { - //open existing cache in existing shared memory - if (db->shmem_cached_fd <= 0) + if (openCache(db) != 0) { - if (openCache(db) != 0) - { - Kdb_unlock(&db->shmem_info->cache_rwlock); - return PERS_COM_FAILURE; - } + return PERS_COM_FAILURE; } - void* val = db->tbl->get(db->tbl, tmp_key, &size); + + setMemoryAddress(db->sharedCache, db->tbl[0]); + + val = db->tbl[0]->get(db->tbl[0], metaKey, &size); if (val == NULL) { - size_read = PERS_COM_ERR_NOT_FOUND; - key_not_found = Kdb_true; + bytesRead = PERS_COM_ERR_NOT_FOUND; + keyNotFound = Kdb_true; } else { @@ -1719,112 +1852,89 @@ sint_t getFromCache(KISSDB* db, void* tmp_key, void* readBuffer, sint_t bufsize, ptr = ptr + sizeof(pers_lldb_cache_flag_e); datasize = *(int*) ptr; ptr = ptr + sizeof(int); //move pointer to beginning of data - size_read = datasize; + bytesRead = datasize; //get data if needed if (!sizeOnly) { if (bufsize < datasize) { - Kdb_unlock(&db->shmem_info->cache_rwlock); return PERS_COM_FAILURE; } else + { (void) memcpy(readBuffer, ptr, datasize); + } } } else { - size_read = PERS_COM_ERR_NOT_FOUND; - key_deleted = Kdb_true; + bytesRead = PERS_COM_ERR_NOT_FOUND; + keyDeleted = Kdb_true; } free(val); } } else - cache_empty = Kdb_true; + { + cacheEmpty = Kdb_true; + } //only read from file if key was not found in cache and if key was not marked as deleted in cache - if ((cache_empty == Kdb_true && key_deleted == Kdb_false) || key_not_found == Kdb_true) + if ((cacheEmpty == Kdb_true && keyDeleted == Kdb_false) || keyNotFound == Kdb_true) { - Kdb_unlock(&db->shmem_info->cache_rwlock); return PERS_STATUS_KEY_NOT_IN_CACHE; //key not found in cache } else { - Kdb_unlock(&db->shmem_info->cache_rwlock); - return size_read; + return bytesRead; } } - - -sint_t getFromDatabaseFile(KISSDB* db, void* tmp_key, void* readBuffer, pers_lldb_purpose_e purpose, sint_t bufsize, - bool_t sizeOnly) +sint_t getFromDatabaseFile(KISSDB* db, void* metaKey, void* readBuffer, sint_t bufsize) { - sint_t size_read = 0; - int datasize = 0; - char* ptr; - char m_data[sizeof(Data_LocalDB_s)] = { 0 }; //temporary buffer that gets filled with read in KISSDB_get + int kdbState = 0; + sint_t bytesRead = 0; + uint32_t size = 0; - int kissdb_status = KISSDB_get(db, tmp_key, m_data); - if (kissdb_status == 0) + kdbState = KISSDB_get(db, metaKey, readBuffer, bufsize, &size); + if (kdbState == 0) { - if (purpose == PersLldbPurpose_DB) - { - ptr = m_data; - ptr += PERS_DB_MAX_SIZE_KEY_DATA; - datasize = *(int*) ptr; - if (!sizeOnly) - { - if (bufsize < datasize) - return PERS_COM_FAILURE; - else - (void) memcpy(readBuffer, m_data, datasize); - } - size_read = datasize; - } - else - { - if (!sizeOnly) - { - if (bufsize < datasize) - return PERS_COM_FAILURE; - else - (void) memcpy(readBuffer, m_data, sizeof(PersistenceConfigurationKey_s)); - } - size_read = sizeof(PersistenceConfigurationKey_s); - } + bytesRead = size; } else { - if (kissdb_status == 1) + if (kdbState == 1) { DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("not found, retval=<"); DLT_INT(kissdb_status); DLT_STRING(">")); - size_read = PERS_COM_ERR_NOT_FOUND; + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("not found, retval=<"); DLT_INT(kdbState); DLT_STRING(">")); + bytesRead = PERS_COM_ERR_NOT_FOUND; } else { DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(tmp_key); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kissdb_status); DLT_STRING(">")); + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("Error with retval=<"); DLT_INT(kdbState); DLT_STRING(">")); } - size_read = PERS_COM_ERR_NOT_FOUND; + bytesRead = PERS_COM_ERR_NOT_FOUND; } - return size_read; + return bytesRead; } -sint_t putToCache(KISSDB* db, sint_t dataSize, char* tmp_key, void* insert_cached_data) +sint_t putToCache(KISSDB* db, sint_t dataSize, char* metaKey, void* cachedData) { - sint_t size_written = 0; - Kdb_wrlock(&db->shmem_info->cache_rwlock); + sint_t bytesWritten = 0; + + //DO NOT ALLOW WRITING TO CACHE IF DATABASE IS OPENED IN READONLY MODE + if(KISSDB_OPEN_MODE_RDONLY == db->shared->openMode ) + { + return PERS_COM_ERR_READONLY; + } //if cache not already created - if (db->shmem_info->cache_initialised == Kdb_false) + if (db->shared->cacheCreated == Kdb_false) { if (createCache(db) != 0) { - Kdb_unlock(&db->shmem_info->cache_rwlock); return PERS_COM_FAILURE; } } @@ -1832,110 +1942,295 @@ sint_t putToCache(KISSDB* db, sint_t dataSize, char* tmp_key, void* insert_cache { if (openCache(db) != 0) { - Kdb_unlock(&db->shmem_info->cache_rwlock); return PERS_COM_FAILURE; } } - + // update db->sharedCache pointer (process adress range mapping can be different after remap) (use + //printf("setMemoryAddress(db->sharedCache, db->tbl[0] = %p \n", db->sharedCache ); + setMemoryAddress(db->sharedCache, db->tbl[0]); //address to first hashtable //put in cache - if (db->tbl->put(db->tbl, tmp_key, insert_cached_data, - sizeof(pers_lldb_cache_flag_e) + sizeof(int) + (size_t) dataSize) == false) //store flag , datasize and data as value in cache + if (db->tbl[0]->put(db->tbl[0], metaKey, cachedData, sizeof(pers_lldb_cache_flag_e) + sizeof(int) + (size_t) dataSize) == + false) //store flag , datasize and data as value in cache { - size_written = PERS_COM_FAILURE; - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Error: Failed to put data into cache")); + bytesWritten = PERS_COM_FAILURE; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Failed to put data into cache: "); DLT_STRING(strerror(errno))); + +#if 0 + int k = 0; + //if additional caches were created by another process -> get referemnce to them and init the local tbl struct + if(db->cacheReferenced < db->shared->cacheCount) + { + printf("additional caches found! \n"); + //init local tbl2 array (internally a malloc occurs) + for(k=1; k < db->shared->cacheCount; k++ ) + { + printf("qhasharr init cache no.= %d with zero \n",k); + db->tbl[k] = qhasharr(db->sharedCache + (k * PERS_CACHE_MEMSIZE) , 0); + db->cacheReferenced++; + } + } + + //reset addresses for all caches mapped to this process space + for(k=0; k < db->shared->cacheCount; k++ ) + { + printf("setMemoryAddress k= %d -- putToCache ptr: %p \n", k, db->sharedCache + (k * PERS_CACHE_MEMSIZE)); + setMemoryAddress(db->sharedCache + (k * PERS_CACHE_MEMSIZE ), db->tbl[k]); + } + + int putOk = 0; + k=0; + //try to insert data until empty slot or same key is found in all of the caches + while (k < db->shared->cacheCount) //store flag , datasize and data as value in cache + { + printf("start while k = %d, cacheCount= %d\n", k, db->shared->cacheCount); + if( db->tbl[k]->put(db->tbl[k], metaKey, cachedData, sizeof(pers_lldb_cache_flag_e) + sizeof(int) + (size_t) dataSize) == true) + { + putOk = 1; + printf("INSERT OK \n"); + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("INSERT INTO Cache Nr: "); DLT_INT(k); DLT_STRING("worked : ")); + break; + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("INSERT INTO Cache Nr: "); DLT_INT(k); DLT_STRING("failed : "); DLT_STRING(strerror(errno))); + } + printf("end while k = %d \n", k); + k++; + } + + + + //if all caches are full -> add new cache and insert data here + if (putOk == 0) + { + printf("data->maxslots 1 = %d \n", db->tbl[0]->data->maxslots); + printf("start adding new cache \n"); + addCache(db); //add additional cache + printf("end adding new cache \n"); + printf("data->maxslots 2 = %d \n", db->tbl[0]->data->maxslots); + + printf("Put not ok -> try to use added cache no. = %d \n", db->shared->cacheCount -1); + + if (db->tbl[db->shared->cacheCount - 1]->put(db->tbl[db->shared->cacheCount - 1], metaKey, cachedData, + sizeof(pers_lldb_cache_flag_e) + sizeof(int) + (size_t) dataSize) == false) //store flag , datasize and data as value in cache + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("INSERT NEW DATA INTO RESIZE OF CACHE FAILED : "); DLT_STRING(strerror(errno))); + + } + else + DLT_LOG(persComLldbDLTCtx, DLT_LOG_DEBUG, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("INSERT NEW DATA INTO RESIZE OF CACHE WORKS : ")); + } + printf("END ----------- \n\n"); +#endif } else - size_written = dataSize; - - Kdb_unlock(&db->shmem_info->cache_rwlock); - return size_written; + { + bytesWritten = dataSize; // return only size of data that has to be stored + } + return bytesWritten; } +sint_t deleteFromCache(KISSDB* db, char* metaKey) +{ + char* ptr; + Data_Cached_s dataCached = { 0 }; + int datasize = 0; + int status = PERS_COM_FAILURE; + Kdb_bool found = Kdb_true; + pers_lldb_cache_flag_e eFlag; + sint_t bytesDeleted = 0; + size_t size = 0; + void* val; + + //DO NOT ALLOW WRITING TO CACHE IF DATABASE IS OPENED IN READONLY MODE + if (KISSDB_OPEN_MODE_RDONLY != db->shared->openMode) + { + dataCached.eFlag = CachedDataDelete; + dataCached.m_dataSize = 0; + + //if cache not already created + if (db->shared->cacheCreated == Kdb_false) + { + if (createCache(db) != 0) + { + return PERS_COM_FAILURE; + } + } + else //open cache + { + if (openCache(db) != 0) + { + return PERS_COM_FAILURE; + } + } + + setMemoryAddress(db->sharedCache, db->tbl[0]); + + val = db->tbl[0]->get(db->tbl[0], metaKey, &size); + if (NULL != val) //check if key to be deleted is in Cache + { + ptr = val; + eFlag = (pers_lldb_cache_flag_e) *(int*) ptr; + ptr += sizeof(int); + datasize = *(int*) ptr; + + //Mark data in cache as deleted + if (eFlag != CachedDataDelete) + { + if (db->tbl[0]->put(db->tbl[0], metaKey, &dataCached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) //do not store any data + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Failed to mark data in cache as deleted")); + bytesDeleted = PERS_COM_ERR_NOT_FOUND; + found = Kdb_false; + } + else + { + bytesDeleted = datasize; + } + } + } + else //check if key to be deleted is in database file + { + //get dataSize + uint32_t size; + status = KISSDB_get(db, metaKey, NULL, 0, &size); + if (status == 0) + { + if (db->tbl[0]->put(db->tbl[0], metaKey, &dataCached, sizeof(pers_lldb_cache_flag_e) + sizeof(int)) == false) + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Failed to mark existing data as deleted")); + bytesDeleted = PERS_COM_ERR_NOT_FOUND; + } + else + { + bytesDeleted = size; + } + } + else + { + if (status == 1) + { + found = Kdb_false; + DLT_LOG(persComLldbDLTCtx, DLT_LOG_WARN, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("not found, retval=<"); DLT_INT(status); DLT_STRING(">")); + } + else + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, + DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("KISSDB_get: key=<"); DLT_STRING(metaKey); DLT_STRING(">, "); DLT_STRING("failed with retval=<"); DLT_INT(status); DLT_STRING(">")); + } + } + } + + if (found == Kdb_false) + { + bytesDeleted = PERS_COM_ERR_NOT_FOUND; + } + } + else + { + bytesDeleted = PERS_COM_ERR_READONLY; + } + return bytesDeleted; +} + sint_t getListandSize(KISSDB* db, pstr_t buffer, sint_t size, bool_t bOnlySizeNeeded, pers_lldb_purpose_e purpose) { - KISSDB_Iterator dbi; - int keycount_file = 0, keycount_cache = 0, result = 0, x = 0, idx = 0, max = 0, used = 0, obj_count, keylength = 0; + char* memory = NULL; + char* ptr; char** tmplist = NULL; + int keyCountFile = 0, keyCountCache = 0, result = 0, x = 0, idx = 0, max = 0, used = 0, objCount = 0; + KISSDB_Iterator dbi; + pers_lldb_cache_flag_e eFlag; + qhasharr_t* tbl; qnobj_t obj; sint_t availableSize = size; - char* ptr; - char* memory = NULL; void* pt; - pers_lldb_cache_flag_e eFlag; - if (purpose == PersLldbPurpose_RCT) - keylength = PERS_RCT_MAX_LENGTH_RESOURCE_ID; - else - keylength = PERS_DB_MAX_LENGTH_KEY_NAME; + int keylength = (purpose == PersLldbPurpose_RCT) ? PERS_RCT_MAX_LENGTH_RESOURCE_ID : PERS_DB_MAX_LENGTH_KEY_NAME; + char kbuf[PERS_DB_MAX_LENGTH_KEY_NAME]; //open existing cache if present and look for keys - if (db->shmem_info->cache_initialised == Kdb_true) + if (db->shared->cacheCreated == Kdb_true) { if (openCache(db) != 0) { - Kdb_unlock(&db->shmem_info->cache_rwlock); return PERS_COM_FAILURE; } else { - obj_count = db->tbl->size(db->tbl, &max, &used); - if (obj_count > 0) + setMemoryAddress(db->sharedCache, db->tbl[0]); + + objCount = db->tbl[0]->size(db->tbl[0], &max, &used); + if (objCount > 0) { - tmplist = malloc(sizeof(char*) * obj_count); + tmplist = malloc(sizeof(char*) * objCount); if (tmplist != NULL) { - while (db->tbl->getnext(db->tbl, &obj, &idx) == true) + while (db->tbl[0]->getnext(db->tbl[0], &obj, &idx) == true) { + size_t keyLen = strlen(obj.name); pt = obj.data; eFlag = (pers_lldb_cache_flag_e) *(int*) pt; if (eFlag != CachedDataDelete) { - tmplist[keycount_cache] = (char*) malloc(strlen(obj.name) + 1); - (void) strncpy(tmplist[keycount_cache], obj.name, strlen(obj.name)); - ptr = tmplist[keycount_cache]; - ptr[strlen(obj.name)] = '\0'; - keycount_cache++; + tmplist[keyCountCache] = (char*) malloc(keyLen + 1); + (void) strncpy(tmplist[keyCountCache], obj.name, keyLen); + ptr = tmplist[keyCountCache]; + ptr[keyLen] = '\0'; + keyCountCache++; } } } else + { return PERS_COM_ERR_MALLOC; + } } } } //look for keys in database file + //Initialise database iterator KISSDB_Iterator_init(db, &dbi); - char kbuf[keylength]; - //get number of keys, stored in database file while (KISSDB_Iterator_next(&dbi, &kbuf, NULL) > 0) - keycount_file++; + { + keyCountFile++; + } - if ((keycount_cache + keycount_file) > 0) + if ((keyCountCache + keyCountFile) > 0) { - int memsize = qhasharr_calculate_memsize(keycount_cache + keycount_file); + int memsize = qhasharr_calculate_memsize(keyCountCache + keyCountFile); //create hashtable that stores the list of keys without duplicates memory = malloc(memsize); if (memory != NULL) { memset(memory, 0, memsize); - qhasharr_t *tbl = qhasharr(memory, memsize); + tbl = qhasharr(memory, memsize); if (tbl == NULL) + { return PERS_COM_ERR_MALLOC; - + } //put keys in cache to a hashtable - for (x = 0; x < keycount_cache; x++) + for (x = 0; x < keyCountCache; x++) { if (tbl->put(tbl, tmplist[x], "0", 1) == true) { if (tmplist[x] != NULL) + { free(tmplist[x]); + } } } free(tmplist); @@ -1947,7 +2242,9 @@ sint_t getListandSize(KISSDB* db, pstr_t buffer, sint_t size, bool_t bOnlySizeNe { size_t keyLen = strnlen(kbuf, sizeof(kbuf)); if (keyLen > 0) + { tbl->put(tbl, kbuf, "0", 1); + } } //count needed size for buffer / copy keys to buffer @@ -1972,82 +2269,132 @@ sint_t getListandSize(KISSDB* db, pstr_t buffer, sint_t size, bool_t bOnlySizeNe free(memory); } else + { return PERS_COM_ERR_MALLOC; + } } return result; } - - - int createCache(KISSDB* db) { - Kdb_bool shmem_creator; + Kdb_bool shmCreator; int status = -1; - db->shmem_cached_fd = kdbShmemOpen(db->shmem_cached_name, PERS_CACHE_MEMSIZE, &shmem_creator); - if (db->shmem_cached_fd != -1) + db->sharedCacheFd = kdbShmemOpen(db->cacheName, PERS_CACHE_MEMSIZE, &shmCreator); + if (db->sharedCacheFd != -1) { - db->shmem_cached = (void*) getKdbShmemPtr(db->shmem_cached_fd, PERS_CACHE_MEMSIZE); - if (db->shmem_cached != ((void *) -1)) - { - db->tbl = qhasharr(db->shmem_cached, PERS_CACHE_MEMSIZE); - if (db->tbl != NULL) + db->sharedCache = (void*) getKdbShmemPtr(db->sharedCacheFd, PERS_CACHE_MEMSIZE); + if (db->sharedCache != ((void*) -1)) + { + // for dynamic cache -> create reference into array db->tbl[0] = qhasharr(db->sharedCache, PERS_CACHE_MEMSIZE); + /* + * Add a function called addCache that resizes shared memory with the size of PERS_CACHE_MEMSIZE + * Then init the additional cache -> db->tbl[n] = qhasharr(db->sharedCache + (n * PERS_CACHE_MEMSIZE) , PERS_CACHE_MEMSIZE) + * Other processes must recognize additional created caches and reopen them with -> db->tbl[0 - n] = qhasharr(db->sharedCache + (n * PERS_CACHE_MEMSIZE) , 0); + * Store the count of created caches in shared information + */ + db->tbl[0] = qhasharr(db->sharedCache, PERS_CACHE_MEMSIZE); + if (db->tbl[0] != NULL) { status = 0; - db->shmem_info->cache_initialised = Kdb_true; + db->shared->cacheCreated = Kdb_true; } } } if (status != 0) - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Error: Failed to create cache")); + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__); DLT_STRING(":"); DLT_STRING("Failed to create cache"); DLT_STRING(strerror(errno))); + } return status; } - int openCache(KISSDB* db) { - Kdb_bool shmem_creator; + Kdb_bool shmCreator; int status = -1; //only open shared memory again if filedescriptor is not initialised yet - if (db->shmem_cached_fd <= 0) //not shared filedescriptor + if (db->sharedCacheFd <= 0) //not shared filedescriptor { - db->shmem_cached_fd = kdbShmemOpen(db->shmem_cached_name, PERS_CACHE_MEMSIZE, &shmem_creator); - if (db->shmem_cached_fd != -1) + db->sharedCacheFd = kdbShmemOpen(db->cacheName, PERS_CACHE_MEMSIZE, &shmCreator); + if (db->sharedCacheFd != -1) { - db->shmem_cached = (void*) getKdbShmemPtr(db->shmem_cached_fd, PERS_CACHE_MEMSIZE); - if (db->shmem_cached != ((void *) -1)) + db->sharedCache = (void*) getKdbShmemPtr(db->sharedCacheFd, PERS_CACHE_MEMSIZE); + if (db->sharedCache != ((void*) -1)) { // use existent hash-table - db->tbl = qhasharr(db->shmem_cached, 0); - if (db->tbl != NULL) + db->tbl[0] = qhasharr(db->sharedCache, 0); + if (db->tbl[0] != NULL) + { status = 0; + } } } } else + { status = 0; + } if (status != 0) - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Error: Failed to open cache")); + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__), DLT_STRING(":"), DLT_STRING("Failed to open cache")); + } return status; } +#if 0 +int addCache(KISSDB* db) +{ + //ftruncate db->sharedCacheFd (oldsize + additionalspace) + + //printf("start ftruncate new cache db->tbl[0]->data->maxslots = %d\n", db->tbl[0]->data->maxslots); + if( ftruncate(db->sharedCacheFd, db->shared->cacheSize + PERS_CACHE_MEMSIZE) < 0) + { + printf("Cache resize failed: %s \n", strerror(errno)); + } + //printf("end ftruncate new cache db->tbl[0]->data->maxslots = %d\n", db->tbl[0]->data->maxslots); + + //mremap oldsize , newsize + //store new cache pointer for this process in db->sharedCache + db->sharedCache = mremap(db->sharedCache, db->shared->cacheSize , db->shared->cacheSize + PERS_CACHE_MEMSIZE, MREMAP_MAYMOVE ); + if (db->sharedCache == MAP_FAILED) + { + printf("cacheresize MAP_FAILED \n"); + } + //printf("end mremap new cache db->tbl[0]->data->maxslots = %d\n", db->tbl[0]->data->maxslots); + //printf("adding cache into db->tbl[%d],----- ptr: %p \n",db->shared->cacheCount, db->sharedCache + (db->shared->cacheCount * PERS_CACHE_MEMSIZE)); + + db->tbl[db->shared->cacheCount] = qhasharr(db->sharedCache + (db->shared->cacheCount * PERS_CACHE_MEMSIZE) , PERS_CACHE_MEMSIZE); + db->cacheReferenced++; + //printf("here 1 \n "); + + //printf("end qhasharr new cache db->tbl[0]->data->maxslots = %d\n", db->tbl[0]->data->maxslots); + + //store new size in shared memory + db->shared->cacheSize += PERS_CACHE_MEMSIZE; + db->shared->cacheCount++; + // add check if cache was resized (check local mapped size versus shared mapped size) for all processes when cache is accessed and do a remap for new size without truncation + return 0; +} +#endif + + int closeCache(KISSDB* db) { int status = -1; - if (kdbShmemClose(db->shmem_cached_fd, db->shmem_cached_name) != Kdb_false) + if (kdbShmemClose(db->sharedCacheFd, db->cacheName) != Kdb_false) { - free(db->shmem_cached_name); //free memory for name obtained by kdbGetShmName() function - if (freeKdbShmemPtr(db->shmem_cached, PERS_CACHE_MEMSIZE) != Kdb_false) + if (freeKdbShmemPtr(db->sharedCache, PERS_CACHE_MEMSIZE) != Kdb_false) + { status = 0; + } } if (status != 0) - DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, - DLT_STRING(__FUNCTION__); DLT_STRING(":");DLT_STRING("Error: Failed to close cache")); + { + DLT_LOG(persComLldbDLTCtx, DLT_LOG_ERROR, DLT_STRING(__FUNCTION__), DLT_STRING(":"), DLT_STRING("Failed to close cache")); + } return status; } diff --git a/src/pers_local_shared_db_access.c b/src/pers_local_shared_db_access.c index a74f82c..638a2f4 100644 --- a/src/pers_local_shared_db_access.c +++ b/src/pers_local_shared_db_access.c @@ -1,8 +1,10 @@ /********************************************************************************************************************** * * Copyright (C) 2012 Continental Automotive Systems, Inc. +* Copyright (C) 2014 XS Embedded GmbH * * Author: Ionut.Ieremie@continental-corporation.com +* simon.disch@xse.de * * Implementation of persComDbAccess.h * @@ -27,6 +29,16 @@ #include "persComDbAccess.h" #include "persComErrors.h" +/** +* \brief returns the max DB key data size +* +* \return the size +*/ +int persComDbgetMaxKeyValueSize(void) +{ + return PERS_DB_MAX_SIZE_KEY_DATA; +} + /** * \brief Obtain a handler to DB indicated by dbPathname * \note : DB is created if it does not exist and (bForceCreationIfNotPresent != 0) diff --git a/test/Makefile.am b/test/Makefile.am index 72fb8d7..6216cc4 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -4,14 +4,13 @@ if DEBUG AM_CFLAGS =$(DEPS_CFLAGS) $(CHECK_CFLAGS) -g else AM_CFLAGS = $(DEPS_CFLAGS) $(CHECK_CFLAGS) -#AM_CFLAGS = -fprofile-arcs -ftest-coverage $(DEPS_CFLAGS) $(CHECK_CFLAGS) endif -noinst_PROGRAMS = persistence_common_object_test +noinst_PROGRAMS = test_pco_key_value_store -persistence_common_object_test_SOURCES = persistence_common_object_test.c -persistence_common_object_test_LDADD = $(DLT_LIBS) $(DEPS_LIBS) $(CHECK_LIBS)\ +test_pco_key_value_store_SOURCES = test_pco_key_value_store.c +test_pco_key_value_store_LDADD = $(DLT_LIBS) $(DEPS_LIBS) $(CHECK_LIBS)\ $(top_srcdir)/src/libpers_common.la -TESTS=persistence_common_object_test +TESTS=test_pco_key_value_store diff --git a/test/test_pco_key_value_store.c b/test/test_pco_key_value_store.c new file mode 100644 index 0000000..5182079 --- /dev/null +++ b/test/test_pco_key_value_store.c @@ -0,0 +1,3135 @@ +/****************************************************************************** + * Project persistence key value store + * (c) copyright 2014 + * Company XS Embedded GmbH + *****************************************************************************/ +/****************************************************************************** + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +******************************************************************************/ +/** +* @file persistence_common_object_test.c +* @ingroup persistency +* @author Simon Disch +* @brief test of persistence key value store +* @see +*/ + +#include +#include +#include +#include +#include /* exit */ +#include +#include +#include + +#include +#include +#include <../inc/protected/persComRct.h> +#include <../inc/protected/persComDbAccess.h> +//#include <../test/pers_com_test_base.h> +//#include <../test/pers_com_check.h> +#include +#include + +#define BUF_SIZE 64 +#define NUM_OF_FILES 3 +#define READ_SIZE 1024 +#define MaxAppNameLen 256 + +/// application id +char gTheAppId[MaxAppNameLen] = { 0 }; + +// definition of weekday +char* dayOfWeek[] = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; + + + +void data_setup(void) +{ + //ssd + DLT_REGISTER_APP("PCOt", "tests the persistence common object library"); +} + +void data_teardown(void) +{ + DLT_UNREGISTER_APP(); +} + + + +START_TEST(test_OpenLocalDB) +{ + int k=0, handle = 0; + int databases = 20; + char path[128]; + int handles[100] = { 0 }; + + persComDbgetMaxKeyValueSize(); + + //Cleaning up testdata folder + remove("/tmp/open-localdb.db"); + remove("/tmp/open-write-cached.db"); + remove("/tmp/open-consecutive.db"); + remove("/tmp/open-write-through.db"); + + int ret = 0; + int ret2 = 0; + + ret = persComDbOpen("/tmp/open-localdb.db", 0x0); //Do not create test.db / only open if present (cached) + fail_unless(ret < 0, "Open open-localdb.db works, but should fail: retval: [%d]", ret); + + ret = persComDbOpen("/tmp/open-localdb.db", 0x0); //Do not create test.db / only open if present + fail_unless(ret < 0, "Open open-localdb.db works, but should fail: retval: [%d]", ret); + + ret = persComDbOpen("/tmp/open-localdb.db", 0x1); //create test.db if not present (cached) + fail_unless(ret >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + ret = persComDbClose(ret); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database: retval: [%d]", ret); + + //test to use more than 16 static handles + for (k = 0; k < databases; k++) + { + snprintf(path, 128, "/tmp/handletest-%d.db", k); + //Cleaning up testdata folder + remove(path); + } + + for (k = 0; k < databases; k++) + { + snprintf(path, 128, "/tmp/handletest-%d.db", k); + handle = persComDbOpen(path, 0x1); //create test.db if not present + handles[k] = handle; + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + } + + //printf("closing! \n"); + for (k = 0; k < databases; k++) + { + ret = persComDbClose(handles[k]); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database with number %d: retval: [%d]", k, ret); + } + + //Test two consecutive open calls + ret = persComDbOpen("/tmp/open-consecutive.db", 0x1); //create test.db if not present (cached) + //printf("TEST handle first: %d \n", ret); + fail_unless(ret >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + ret2 = persComDbOpen("/tmp/open-consecutive.db", 0x1); //create test.db if not present (cached) + //printf("TEST handle second: %d \n", ret2); + fail_unless(ret2 >= 0, "Failed at consecutive open: retval: [%d]", ret2); + + ret = persComDbClose(ret); + if (ret != 0) + { + printf("persComDbClose() 1 failed: [%d] \n", ret); + } + ret = persComDbClose(ret2); + if (ret != 0) + { + printf("persComDbClose() 2 failed: [%d] \n", ret); + } + + + //############# test write through ######################### + //first create database file + ret2 = persComDbOpen("/tmp/open-write-through.db", 0x3); //write through and create test.db if not present + //printf("handle 2: %d \n", ret2); + fail_unless(ret2 >= 0, "Failed at write through open: retval: [%d]", ret); + + ret = persComDbClose(ret2); + if (ret != 0) + { + printf("persComDbClose() cached failed: [%d] \n", ret); + } + + //then open existing file in write through mode + ret2 = persComDbOpen("/tmp/open-write-through.db", 0x2); //write through open + //printf("handle 2: %d \n", ret2); + fail_unless(ret2 >= 0, "Failed at write through / open existing: retval: [%d]", ret); + + ret = persComDbClose(ret2); + if (ret != 0) + { + printf("persComDbClose() write through / open existing failed: [%d] \n", ret); + } + //########################################################### + + + + + //############# test cached with no creation forced ######### + //first create the database + ret2 = persComDbOpen("/tmp/open-write-cached.db", 0x1); //write cached create database + //printf("handle 2: %d \n", ret2); + fail_unless(ret2 >= 0, "Failed at write cached / create open: retval: [%d]", ret); + + ret = persComDbClose(ret2); + if (ret != 0) + { + printf("persComDbClose() cached / create failed: [%d] \n", ret); + } + //then try to open existing with cached mode + ret = persComDbOpen("/tmp/open-write-cached.db", 0x0); //cached and DO NOT create database + //printf("handle: %d \n", ret); + fail_unless(ret >= 0, "Failed at open cached / no database creatio: retval: [%d]", ret); //fail if open works, but should fail + + ret = persComDbClose(ret); + if (ret != 0) + printf("persComDbClose() with cached / no database creation -> failed: [%d] \n", ret); + //######################################################### + + + + + + //try to close a non existent database handle + ret = persComDbClose(15); + fail_unless(ret < 0, "Database closing works, but should not: retval: [%d]", ret); + +} +END_TEST + +//START_TEST(test_OpenLocalDB) +//{ +// int handle = 0; +// int ret = 0; +// +// char write2[READ_SIZE] = { 0 }; +// char key[128] = { 0 }; +// int i = 0; +// +// //Cleaning up testdata folder +// remove("/tmp/open-localdb3.db"); +// handle = persComDbOpen("/tmp/open-localdb3.db", 0x1); //create test.db if not present +// fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", handle); +// //write keys to cache +// for(i=0; i< 10; i++) +// { +// snprintf(key, 128, "Key%d",i); +// memset(write2, 0, sizeof(write2)); +// snprintf(write2, 128, "DATA-%d-%d",i,i*i ); +// ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); +// fail_unless(ret == strlen(write2), "Wrong write size"); +// } +// handle = persComDbClose(handle); +// if (handle != 0) +// { +// printf("persComDbClose() failed: [%d] \n", handle); +// } +// fail_unless(handle == 0, "Failed to close database: retval: [%d]", handle); +// +// +// handle = persComDbOpen("/tmp/open-localdb3.db", 0x1); //create test.db if not present +// fail_unless(handle >= 0, "Failed to create non existent lDB 2nd time: retval: [%d]", handle); +// +// handle = persComDbClose(handle); +// if (handle != 0) +// { +// printf("persComDbClose() 2nd time failed: [%d] \n", handle); +// } +// fail_unless(handle == 0, "Failed to close database 2nd time: retval: [%d]", handle); +// +//} +//END_TEST + + +START_TEST(test_OpenRCT) +{ + //Cleaning up testdata folder + remove("/tmp/open-rct.db"); + + int ret1 = 0; + int ret2 = 0; + ret1 = persComRctOpen("/tmp/open-rct.db", 0x0); //Do not create rct.db / only open if present + fail_unless(ret1 < 0, "Open open-rct.db works, but should fail: retval: [%d]", ret1); + + ret2 = persComRctOpen("/tmp/open-rct.db", 0x1); //create test.db if not present (cached) + fail_unless(ret2 >= 0, "Failed to create non existent rct: retval: [%d]", ret2); + + ret1 = persComRctClose(ret1); + ret2 = persComRctClose(ret2); + if (ret2 != 0) + { + printf("persComRctClose() failed: [%d] \n", ret2); + } + fail_unless(ret2 == 0, "Failed to close RCT database: retval: [%d]", ret2); + +} +END_TEST + + + +/* + * Test if a database can be opened in readonly mode + * First, a valid database file gets written. Then the database is opened in readonly mode + * Write to the readonly opened database must fail. + * After reopening, the reading of keys tried to write in readonly mode must fail. + * + */ +START_TEST(test_ReadOnlyDatabase) +{ + int ret = 0; + int handle = 0; + char write2[READ_SIZE] = { 0 }; + char read[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + char sysTimeBuffer[256]; + struct tm* locTime; + int i =0; + + //Cleaning up testdata folder + remove("/tmp/open-readonly.db"); + +#if 1 + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 128, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + handle = persComDbOpen("/tmp/open-readonly.db", 0x1); //create initial database with read write access + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + snprintf(write2, 128, "%s %s", "/key_70", sysTimeBuffer); + + //write to cache + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2) , "Wrong write size while inserting in cache"); + } + + //read from cache + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong read size while reading from cache"); + } + + //printf("read from cache ok \n"); + + //persist data in cache to file + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + + handle = persComDbOpen("/tmp/open-readonly.db", 0x4); //open existing database in readonly mode + fail_unless(handle >= 0, "Failed to reopen existing lDB: retval: [%d]", ret); + + //printf("open in readonly ok \n"); + + //read from database file + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + memset(read, 0, 1024); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + //printf("read: %d returns: %s \n", i, read); + fail_unless(ret == strlen(write2), "Wrong read size"); + } + + //write and delete must fail + ret = persComDbWriteKey(handle, "SHOULD_NOT_BE_PERSISTED", (char*) write2, strlen(write2)); + fail_unless(ret < 0, "Writing to readonly opened database worked, but should fail for key: [SHOULD_NOT_BE_PERSISTED] !"); + + ret = persComDbDeleteKey(handle, "SHOULD_NOT_BE_PERSISTED"); + fail_unless(ret < 0, "Deletion in readonly opened database worked, but should fail for key: [SHOULD_NOT_BE_PERSISTED] !"); + + //writeback should not be invoked because of readonly mode + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + + handle = persComDbOpen("/tmp/open-readonly.db", 0x4); //open in readonly mode + fail_unless(handle >= 0, "Failed to reopen existing lDB: retval: [%d]", ret); + + memset(read, 0, 1024); + ret = persComDbReadKey(handle, "SHOULD_NOT_BE_PERSISTED", (char*) read, strlen(write2)); + //printf("read: %d returns: %s \n", i, read); + fail_unless(ret < 0, "Reading of key: [SHOULD_NOT_BE_PERSISTED] works, but should fail!"); + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); +#endif +} +END_TEST + + + + + + + + + + + +/* + * Write data to a key using the key interface in local DB. + * First write data to different keys and after + * read the data for verification. + */ +START_TEST(test_SetDataLocalDB) +{ + int ret = 0; + int handle = 0; + char write2[READ_SIZE] = { 0 }; + char read[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + char sysTimeBuffer[256]; + struct tm* locTime; + int i =0; + + //Cleaning up testdata folder + remove("/tmp/write-localdb.db"); + +#if 1 + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 128, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + handle = persComDbOpen("/tmp/write-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + snprintf(write2, 128, "%s %s", "/key_70", sysTimeBuffer); + + //write to cache + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2) , "Wrong write size while inserting in cache"); + } + + //read from cache + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong read size while reading from cache"); + } + + //printf("read from cache ok \n"); + + //persist data in cache to file + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + + handle = persComDbOpen("/tmp/write-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to reopen existing lDB: retval: [%d]", ret); + + + //printf("open ok \n"); + + //read from database file + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + memset(read, 0, 1024); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + + //printf("read: %d returns: %s \n", i, read); + fail_unless(ret == strlen(write2), "Wrong read size"); + } + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +#endif +} +END_TEST + + + + +/* + * Write data to a key using the key interface in local DB. + * First the data gets written to Cache. Then this data is read from the Cache. + * After that, the database gets closed in order to persist the cached data to the database file + * The database file is opened again and the keys are read from file for verification + */ +START_TEST(test_GetDataLocalDB) +{ + int ret = 0; + int handle = 0; + unsigned char readBuffer[READ_SIZE] = { 0 }; + char write2[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + int i = 0; + + //Cleaning up testdata folder + remove("/tmp/get-localdb.db"); + + +#if 1 + + handle = persComDbOpen("/tmp/get-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + + //write keys to cache + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key%d",i); + memset(write2, 0, sizeof(write2)); + snprintf(write2, 128, "DATA-%d-%d",i,i*i ); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong write size"); + } + + //Read keys from cache + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key%d",i); + memset(write2, 0, sizeof(write2)); + snprintf(write2, sizeof(write2), "DATA-%d-%d",i,i*i ); + memset(readBuffer, 0, sizeof(readBuffer)); + ret = persComDbReadKey(handle, key, (char*) readBuffer, sizeof(readBuffer)); + fail_unless(ret == strlen(write2), "Wrong read size"); + fail_unless(memcmp(readBuffer, write2, sizeof(readBuffer)) == 0, "Reading Data from Cache failed: Buffer not correctly read"); + } + + //persist changed data for this lifecycle + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: Cached Data was not written back: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + //open database again + handle = persComDbOpen("/tmp/get-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to reopen existing lDB: retval: [%d]", ret); + + + //Read keys from database file + for(i=0; i< 300; i++) + { + snprintf(key, 128, "Key%d",i); + memset(write2, 0, sizeof(write2)); + snprintf(write2, sizeof(write2), "DATA-%d-%d",i,i*i ); + memset(readBuffer, 0, sizeof(readBuffer)); + ret = persComDbReadKey(handle, key, (char*) readBuffer, sizeof(readBuffer)); + fail_unless(ret == strlen(write2), "Wrong read size"); + fail_unless(memcmp(readBuffer, write2, sizeof(readBuffer)) == 0, "Reading Data from File failed: Buffer not correctly read"); + } + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +#endif +} +END_TEST + + + + +/* + * Get the size from an existing local DB needed to store all already inserted key names in a list + * First insert 3 different keys then close the database to persist the data an reopen it. + * Then get the size of all keys separated with '\0' + * After that a duplicate key gets written to the cache and the size of the list is read again to test if duplicate keys are ignored correctly. + * Then a new key gets written to the cache and the list size is read again to test if keys in cache and also keys in the database file are counted. + */ +START_TEST(test_GetKeyListSizeLocalDB) +{ + int ret = 0; + int handle = 0; + char write1[READ_SIZE] = { 0 }; + char write2[READ_SIZE] = { 0 }; + char sysTimeBuffer[256]; + int listSize = 0; + char key[8] = { 0 }; + struct tm* locTime; + + //Cleaning up testdata folder + remove("/tmp/localdb-size-keylist.db"); + + +#if 1 + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 128, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + handle = persComDbOpen("/tmp/localdb-size-keylist.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB for keylist test: retval: [%d]", ret); + + + snprintf(key, 8, "%s", "key_123"); + ret = persComDbWriteKey(handle, key, (char*) sysTimeBuffer, strlen(sysTimeBuffer)); + fail_unless(ret == strlen(sysTimeBuffer), "Wrong write size"); + + + snprintf(key, 8, "%s", "key_456"); + snprintf(write1, 128, "%s %s", "k_456", sysTimeBuffer); + ret = persComDbWriteKey(handle, key, (char*) write1, strlen(write1)); + fail_unless(ret == strlen(write1), "Wrong write size"); + + snprintf(key, 8, "%s", "key_789"); + snprintf(write2, 128, "%s %s", "k_789", sysTimeBuffer); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong write size"); + + listSize = persComDbGetSizeKeysList(handle); + //printf("LISTSIZE: %d \n", listSize); + fail_unless(listSize == 3 * strlen(key) + 3, "Wrong list size read from cache"); + + //persist changes in order to read only keys that are in database file + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + handle = persComDbOpen("/tmp/localdb-size-keylist.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB for keylist test: retval: [%d]", ret); + + + //write duplicated key to cache + snprintf(key, 8, "%s", "key_789"); + snprintf(write2, 128, "%s %s", "k_789", sysTimeBuffer); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong write size"); + + //get needed listsize (looks for keys in cache and in database file) duplicate keys occuring in cache AND in database file are removed + // listsize here must be 24 + listSize = persComDbGetSizeKeysList(handle); + //printf("LISTSIZE: %d \n", listSize); + fail_unless(listSize == 3 * strlen(key) + 3, "Wrong list size read from file"); + + //write new key to cache + snprintf(key, 8, "%s", "key_000"); + snprintf(write2, 128, "%s %s", "k_000", sysTimeBuffer); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + + //read list size again (must be 32) + listSize = persComDbGetSizeKeysList(handle); + //printf("LISTSIZE: %d \n", listSize); + fail_unless(listSize == 4 * strlen(key) + 4, "Wrong list size read from combined cache / file"); + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +#endif + +} +END_TEST + + + + +/* Get the resource list from an existing local database with already inserted key names + * Insert some keys then get the list of all keys separated with '\0' + * Then check if the List returned contains all of the keys inserted before + */ +START_TEST(test_GetKeyListLocalDB) +{ + int ret = 0; + int handle = 0; + char write1[READ_SIZE] = { 0 }; + char write2[READ_SIZE] = { 0 }; + char sysTimeBuffer[256]; + char origKeylist[256] = { 0 }; + char key1[8] = { 0 }; + char key2[8] = { 0 }; + char key3[8] = { 0 }; + char key4[8] = { 0 }; + struct tm* locTime; + char* keyList = NULL; + int listSize = 0; + + //Cleaning up testdata folder + remove("/tmp/localdb-keylist.db"); + +#if 1 + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 128, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + handle = persComDbOpen("/tmp/localdb-keylist.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB for keylist test: retval: [%d]", ret); + + snprintf(key1, 8, "%s", "key_123"); + ret = persComDbWriteKey(handle, key1, (char*) sysTimeBuffer, strlen(sysTimeBuffer)); + fail_unless(ret == strlen(sysTimeBuffer), "Wrong write size"); + + snprintf(key2, 8, "%s", "key_456"); + snprintf(write1, 128, "%s %s", "k_456", sysTimeBuffer); + ret = persComDbWriteKey(handle, key2, (char*) write1, strlen(write1)); + fail_unless(ret == strlen(write1), "Wrong write size"); + + snprintf(key3, 8, "%s", "key_789"); + snprintf(write2, 128, "%s %s", "k_789", sysTimeBuffer); + ret = persComDbWriteKey(handle, key3, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong write size"); + + //close database in order to persist the cached keys. + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + handle = persComDbOpen("/tmp/localdb-keylist.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB for keylist test: retval: [%d]", ret); + + + //write to cache + snprintf(key4, 8, "%s", "key_456"); + snprintf(write1, 128, "%s %s", "k_456", sysTimeBuffer); + ret = persComDbWriteKey(handle, key4, (char*) write1, strlen(write1)); + + //read keys from file and from cache + listSize = persComDbGetSizeKeysList(handle); + fail_unless(listSize == 3 * strlen(key1) + 3, "Wrong list size"); + + keyList = (char*) malloc(listSize); + ret = persComDbGetKeysList(handle, keyList, listSize); + int cmp_result = 0; + + //try all possible key orders in the list + snprintf(origKeylist, 24, "%s%c%s%c%s", key1, '\0', key2, '\0', key3); + if( memcmp(keyList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key1, '\0', key3, '\0', key2); + if(memcmp(keyList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key2, '\0', key3, '\0', key1); + if(memcmp(keyList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key2, '\0', key1, '\0', key3); + if(memcmp(keyList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key3, '\0', key1, '\0', key2); + if(memcmp(keyList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key3, '\0', key2, '\0', key1); + if(memcmp(keyList, origKeylist, listSize) != 0) + { + cmp_result = 1; + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + + // printf("original keylist: [%s] \n", origKeylist); + // printf("keylist: [%s] \n", keyList); + free(keyList); + fail_unless(cmp_result == 0, "List not correctly read"); + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +#endif + +} +END_TEST + +/* + * Get the size from an existing RCT database needed to store all already inserted key names in a list + * Insert some keys then get the size of all keys separated with '\0'. Then close the database and reopen it to read the size again (from file). + * After that, close and reopen the database, to insert a duplicate key and a new key into cache. Then verify the listsize again. + */ +START_TEST(test_GetResourceListSizeRct) +{ + int ret = 0; + int handle = 0; + char sysTimeBuffer[256]; + char key1[8] = { 0 }; + char key2[8] = { 0 }; + char key3[8] = { 0 }; + char key4[8] = { 0 }; + int listSize = 0; + struct tm* locTime; + + PersistenceConfigurationKey_s psConfig; + psConfig.policy = PersistencePolicy_wt; + psConfig.storage = PersistenceStorage_local; + psConfig.type = PersistenceResourceType_key; + psConfig.permission = PersistencePermission_ReadWrite; + + //Cleaning up testdata folder + remove("/tmp/rct-size-resource-list.db"); + +#if 1 + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 64, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + handle = persComRctOpen("/tmp/rct-size-resource-list.db", 0x1); //create rct.db if not present + fail_unless(handle >= 0, "Failed to create non existent rct: retval: [%d]", ret); + + memset(psConfig.custom_name, 0, sizeof(psConfig.custom_name)); + memset(psConfig.customID, 0, sizeof(psConfig.customID)); + memset(psConfig.reponsible, 0, sizeof(psConfig.reponsible)); + + psConfig.max_size = 12345; + char custom_name[PERS_RCT_MAX_LENGTH_CUSTOM_NAME] = "this is the custom name"; + char custom_ID[PERS_RCT_MAX_LENGTH_CUSTOM_ID] = "this is the custom ID"; + char responsible[PERS_RCT_MAX_LENGTH_RESPONSIBLE] = "this is the responsible"; + + strncpy(psConfig.custom_name, custom_name, strlen(custom_name)); + strncpy(psConfig.customID, custom_ID, strlen(custom_ID)); + strncpy(psConfig.reponsible, responsible, strlen(responsible)); + + snprintf(key1, 8, "%s", "key_123"); + ret = persComRctWrite(handle, key1, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + snprintf(key2, 8, "%s", "key_45"); + ret = persComRctWrite(handle, key2, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + snprintf(key3, 8, "%s", "key_7"); + ret = persComRctWrite(handle, key3, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + //get listsize from cache + listSize = persComRctGetSizeResourcesList(handle); + fail_unless(listSize == 3 * strlen(key1), "Read Wrong list size from file and cache"); + + //persist cached data + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + handle = persComRctOpen("/tmp/rct-size-resource-list.db", 0x1); //create rct.db if not present + fail_unless(handle >= 0, "Failed to create non existent rct: retval: [%d]", ret); + + //get listsize from file + listSize = persComRctGetSizeResourcesList(handle); + fail_unless(listSize == 3 * strlen(key1), "Read Wrong list size from file and cache"); + + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database: retval: [%d]", ret); + + handle = persComRctOpen("/tmp/rct-size-resource-list.db", 0x1); //create rct.db if not present + fail_unless(handle >= 0, "Failed to create non existent rct: retval: [%d]", ret); + + //insert duplicate key + snprintf(key3, 8, "%s", "key_7"); + ret = persComRctWrite(handle, key3, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + //insert new key + snprintf(key4, 8, "%s", "key_new"); + ret = persComRctWrite(handle, key4, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + //get listsize if keys are in cache and in file + listSize = persComRctGetSizeResourcesList(handle); + fail_unless(listSize == strlen(key1)+ strlen(key2) + strlen(key3) + strlen(key4) + 4, "Read Wrong list size from file and cache"); + + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +#endif + +} +END_TEST + +/* + * Get the resource list from an existing RCT database with already inserted key names + * Insert some keys then get the list of all keys separated with '\0' + * Then check if the List returned contains all of the keys inserted before + */ +START_TEST(test_GetResourceListRct) +{ + int ret = 0; + int handle = 0; + char sysTimeBuffer[256]; + char origKeylist[256] = { 0 }; + char* resourceList = NULL; + int listSize = 0; + char key1[8] = { 0 }; + char key2[8] = { 0 }; + char key3[8] = { 0 }; + char key4[8] = { 0 }; + struct tm* locTime; + + PersistenceConfigurationKey_s psConfig; + psConfig.policy = PersistencePolicy_wt; + psConfig.storage = PersistenceStorage_local; + psConfig.type = PersistenceResourceType_key; + psConfig.permission = PersistencePermission_ReadWrite; + + //Cleaning up testdata folder + remove("/tmp/rct-resource-list.db"); + +#if 1 + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 64, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + handle = persComRctOpen("/tmp/rct-resource-list.db", 0x1); //create rct.db if not present + fail_unless(handle >= 0, "Failed to create non existent rct: retval: [%d]", ret); + + memset(psConfig.custom_name, 0, sizeof(psConfig.custom_name)); + memset(psConfig.customID, 0, sizeof(psConfig.customID)); + memset(psConfig.reponsible, 0, sizeof(psConfig.reponsible)); + + psConfig.max_size = 12345; + char custom_name[PERS_RCT_MAX_LENGTH_CUSTOM_NAME] = "this is the custom name"; + char custom_ID[PERS_RCT_MAX_LENGTH_CUSTOM_ID] = "this is the custom ID"; + char responsible[PERS_RCT_MAX_LENGTH_RESPONSIBLE] = "this is the responsible"; + + strncpy(psConfig.custom_name, custom_name, strlen(custom_name)); + strncpy(psConfig.customID, custom_ID, strlen(custom_ID)); + strncpy(psConfig.reponsible, responsible, strlen(responsible)); + + snprintf(key1, 8, "%s", "key_123"); + ret = persComRctWrite(handle, key1, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + snprintf(key2, 8, "%s", "key_456"); + ret = persComRctWrite(handle, key2, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + snprintf(key3, 8, "%s", "key_789"); + ret = persComRctWrite(handle, key3, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + snprintf(origKeylist, 24, "%s%c%s%c%s", key1, '\0', key3, '\0', key2); + + //persist keys to file + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + handle = persComRctOpen("/tmp/rct-resource-list.db", 0x1); //create rct.db if not present + fail_unless(handle >= 0, "Failed to create non existent rct: retval: [%d]", ret); + + //write duplicate key to cache + snprintf(key4, 8, "%s", "key_456"); + ret = persComRctWrite(handle, key4, &psConfig); + fail_unless(ret == sizeof(psConfig), "Wrong write size"); + + //read keys from file and from cache + listSize = persComRctGetSizeResourcesList(handle); + fail_unless(listSize == 3 * strlen(key1) + 3, "Wrong list size"); + + resourceList = (char*) malloc(listSize); + ret = persComRctGetResourcesList(handle, resourceList, listSize); + + //compare returned list (unsorted) with original list + int cmp_result = 0; + + snprintf(origKeylist, 24, "%s%c%s%c%s", key1, '\0', key2, '\0', key3); + if( memcmp(resourceList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key1, '\0', key3, '\0', key2); + if(memcmp(resourceList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key2, '\0', key3, '\0', key1); + if(memcmp(resourceList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key2, '\0', key1, '\0', key3); + if(memcmp(resourceList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key3, '\0', key1, '\0', key2); + if(memcmp(resourceList, origKeylist, listSize) != 0) + { + cmp_result = 1; + snprintf(origKeylist, 24, "%s%c%s%c%s", key3, '\0', key2, '\0', key1); + if(memcmp(resourceList, origKeylist, listSize) != 0) + { + cmp_result = 1; + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + } + else + { + cmp_result = 0; + } + + //printf("original resourceList: [%s] \n", origKeylist); + //printf("resourceList: [%s]\n", resourceList); + free(resourceList); + fail_unless(cmp_result == 0, "List not correctly read"); + + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database: retval: [%d]", ret); + +#endif + +} +END_TEST + +/* + * Write data to a key using the key interface for RCT databases + * First write data to different keys and after that + * read the data for verification. + */ +START_TEST(test_SetDataRCT) +{ + int ret = 0; + int handle = 0; + char sysTimeBuffer[256]; + struct tm* locTime; + + PersistenceConfigurationKey_s psConfig, psConfig_out; + psConfig.policy = PersistencePolicy_wt; + psConfig.storage = PersistenceStorage_local; + psConfig.type = PersistenceResourceType_key; + psConfig.permission = PersistencePermission_ReadWrite; + +#if 1 + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 64, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + //Cleaning up testdata folder + remove("/tmp/write-rct.db"); + + handle = persComRctOpen("/tmp/write-rct.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + memset(psConfig.custom_name, 0, sizeof(psConfig.custom_name)); + memset(psConfig.customID, 0, sizeof(psConfig.customID)); + memset(psConfig.reponsible, 0, sizeof(psConfig.reponsible)); + + psConfig.max_size = 12345; + char custom_name[PERS_RCT_MAX_LENGTH_CUSTOM_NAME] = "this is the custom name"; + char custom_ID[PERS_RCT_MAX_LENGTH_CUSTOM_ID] = "this is the custom ID"; + char responsible[PERS_RCT_MAX_LENGTH_RESPONSIBLE] = "this is the responsible"; + + strncpy(psConfig.custom_name, custom_name, strlen(custom_name)); + strncpy(psConfig.customID, custom_ID, strlen(custom_ID)); + strncpy(psConfig.reponsible, responsible, strlen(responsible)); + + +//printf("Custom ID : %s\n", psConfig.customID ); +//printf("Custom Name : %s\n", psConfig.custom_name ); +//printf("reponsible : %s\n", psConfig.reponsible ); +//printf("max_size : %d\n", psConfig.max_size ); +//printf("permission : %d\n", psConfig.permission ); +//printf("type : %d\n", psConfig.type ); +//printf("storage : %d\n", psConfig.storage ); +//printf("policy : %d\n", psConfig.policy ); + + + ret = persComRctWrite(handle, "69", &psConfig); + fail_unless(ret == sizeof(psConfig), "wrong write size \n"); +#if 1 + + memset(psConfig_out.custom_name, 0, sizeof(psConfig_out.custom_name)); + memset(psConfig_out.customID, 0, sizeof(psConfig_out.customID)); + memset(psConfig_out.reponsible, 0, sizeof(psConfig_out.reponsible)); + + //read from cache + ret = persComRctRead(handle, "69", &psConfig_out); + fail_unless(ret == sizeof(psConfig), "Wrong read size from cache"); + + + //persist data in cache to database file + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + //reopen database + handle = persComRctOpen("/tmp/write-rct.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + /* + * now read the data written in the previous steps to the keys in RCT + * and verify data has been written correctly. + */ + memset(psConfig_out.custom_name, 0, sizeof(psConfig_out.custom_name)); + memset(psConfig_out.customID, 0, sizeof(psConfig_out.customID)); + memset(psConfig_out.reponsible, 0, sizeof(psConfig_out.reponsible)); + + //read from file + ret = persComRctRead(handle, "69", &psConfig_out); + fail_unless(ret == sizeof(psConfig), "Wrong read size from file"); + +//printf("Custom ID : %s\n", psConfig_out.customID ); +//printf("Custom Name : %s\n", psConfig_out.custom_name ); +//printf("reponsible : %s\n", psConfig_out.reponsible ); +//printf("max_size : %d\n", psConfig_out.max_size ); +//printf("permission : %d\n", psConfig_out.permission ); +//printf("type : %d\n", psConfig_out.type ); +//printf("storage : %d\n", psConfig_out.storage ); +//printf("policy : %d\n", psConfig_out.policy ); + + fail_unless(strncmp(psConfig.customID, psConfig_out.customID, strlen(psConfig_out.customID)) == 0, + "Buffer not correctly read"); + fail_unless(strncmp(psConfig.custom_name, psConfig_out.custom_name, strlen(psConfig_out.custom_name)) == 0, + "Buffer not correctly read"); + fail_unless(strncmp(psConfig.reponsible, psConfig_out.reponsible, strlen(psConfig_out.reponsible)) == 0, + "Buffer not correctly read"); + fail_unless(psConfig.max_size == psConfig_out.max_size, "Buffer not correctly read"); + fail_unless(psConfig.permission == psConfig_out.permission, "Buffer not correctly read"); + fail_unless(psConfig.policy == psConfig_out.policy, "Buffer not correctly read"); + fail_unless(psConfig.storage == psConfig_out.storage, "Buffer not correctly read"); + fail_unless(psConfig.type == psConfig_out.type, "Buffer not correctly read"); + + //persist to database file + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database: retval: [%d]", ret); + +#endif +#endif + +} +END_TEST + + + +/* + * Test reading of data to a key using the key interface for RCT + * First write data to cache, then read from cache. + * Then the database gets closed and reopened. + * The next read for verification is done from file. + */ +START_TEST(test_GetDataRCT) +{ + int ret = 0; + int handle = 0; + char sysTimeBuffer[256]; + struct tm* locTime; + + PersistenceConfigurationKey_s psConfig, psConfig_out; + psConfig.policy = PersistencePolicy_wt; + psConfig.storage = PersistenceStorage_local; + psConfig.type = PersistenceResourceType_key; + psConfig.permission = PersistencePermission_ReadWrite; + +#if 1 + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 64, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + //Cleaning up testdata folder + remove("/tmp/get-rct.db"); + + handle = persComRctOpen("/tmp/get-rct.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + memset(psConfig.custom_name, 0, sizeof(psConfig.custom_name)); + memset(psConfig.customID, 0, sizeof(psConfig.customID)); + memset(psConfig.reponsible, 0, sizeof(psConfig.reponsible)); + psConfig.max_size = 12345; + + char custom_name[PERS_RCT_MAX_LENGTH_CUSTOM_NAME] = "this is the custom name"; + char custom_ID[PERS_RCT_MAX_LENGTH_CUSTOM_ID] = "this is the custom ID"; + char responsible[PERS_RCT_MAX_LENGTH_RESPONSIBLE] = "this is the responsible"; + + strncpy(psConfig.custom_name, custom_name, strlen(custom_name)); + strncpy(psConfig.customID, custom_ID, strlen(custom_ID)); + strncpy(psConfig.reponsible, responsible, strlen(responsible)); + + + ret = persComRctWrite(handle, "69", &psConfig); + fail_unless(ret == sizeof(psConfig), "write size wrong"); +#if 1 + + + /* + * now read the data written in the previous steps to the keys in RCT + * and verify data has been written correctly. + */ + memset(psConfig_out.custom_name, 0, sizeof(psConfig_out.custom_name)); + memset(psConfig_out.customID, 0, sizeof(psConfig_out.customID)); + memset(psConfig_out.reponsible, 0, sizeof(psConfig_out.reponsible)); + +//read from cache + ret = persComRctRead(handle, "69", &psConfig_out); + fail_unless(ret == sizeof(psConfig_out), "Wrong read size from cache"); + +//printf("Custom ID : %s\n", psConfig_out.customID ); +//printf("Custom Name : %s\n", psConfig_out.custom_name ); +//printf("reponsible : %s\n", psConfig_out.reponsible ); +//printf("max_size : %d\n", psConfig_out.max_size ); +//printf("permission : %d\n", psConfig_out.permission ); +//printf("type : %d\n", psConfig_out.type ); +//printf("storage : %d\n", psConfig_out.storage ); +//printf("policy : %d\n", psConfig_out.policy ); + + + + fail_unless(strncmp(psConfig.customID, psConfig_out.customID, strlen(psConfig_out.customID)) == 0, + "Buffer not correctly read"); + fail_unless(strncmp(psConfig.custom_name, psConfig_out.custom_name, strlen(psConfig_out.custom_name)) == 0, + "Buffer not correctly read"); + fail_unless(strncmp(psConfig.reponsible, psConfig_out.reponsible, strlen(psConfig_out.reponsible)) == 0, + "Buffer not correctly read"); + fail_unless(psConfig.max_size == psConfig_out.max_size, "Buffer not correctly read"); + fail_unless(psConfig.permission == psConfig_out.permission, "Buffer not correctly read"); + fail_unless(psConfig.policy == psConfig_out.policy, "Buffer not correctly read"); + fail_unless(psConfig.storage == psConfig_out.storage, "Buffer not correctly read"); + fail_unless(psConfig.type == psConfig_out.type, "Buffer not correctly read"); + + + //persist to database file + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database: retval: [%d]", ret); + +#endif +#endif + + + handle = persComRctOpen("/tmp/get-rct.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + + memset(psConfig_out.custom_name, 0, sizeof(psConfig_out.custom_name)); + memset(psConfig_out.customID, 0, sizeof(psConfig_out.customID)); + memset(psConfig_out.reponsible, 0, sizeof(psConfig_out.reponsible)); + + + //read from file + ret = persComRctRead(handle, "69", &psConfig_out); + fail_unless(ret == sizeof(psConfig_out), "Wrong read size from file"); + +//printf("Custom ID : %s\n", psConfig_out.customID ); +//printf("Custom Name : %s\n", psConfig_out.custom_name ); +//printf("reponsible : %s\n", psConfig_out.reponsible ); +//printf("max_size : %d\n", psConfig_out.max_size ); +//printf("permission : %d\n", psConfig_out.permission ); +//printf("type : %d\n", psConfig_out.type ); +//printf("storage : %d\n", psConfig_out.storage ); +//printf("policy : %d\n", psConfig_out.policy ); + + fail_unless(strncmp(psConfig.customID, psConfig_out.customID, strlen(psConfig_out.customID)) == 0, + "Buffer not correctly read from file"); + fail_unless(strncmp(psConfig.custom_name, psConfig_out.custom_name, strlen(psConfig_out.custom_name)) == 0, + "Buffer not correctly read from file"); + fail_unless(strncmp(psConfig.reponsible, psConfig_out.reponsible, strlen(psConfig_out.reponsible)) == 0, + "Buffer not correctly read from file"); + fail_unless(psConfig.max_size == psConfig_out.max_size, "Buffer not correctly read from file"); + fail_unless(psConfig.permission == psConfig_out.permission, "Buffer not correctly read from file"); + fail_unless(psConfig.policy == psConfig_out.policy, "Buffer not correctly read from file"); + fail_unless(psConfig.storage == psConfig_out.storage, "Buffer not correctly read from file"); + fail_unless(psConfig.type == psConfig_out.type, "Buffer not correctly read from file"); + + + ret = persComRctClose(handle); + if (ret != 0) + { + printf("persComRctClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database: retval: [%d]", ret); + + +} +END_TEST + + + +/* + * Test to get the datasize for a key + * write a key to cache, then the size of the data to that key from cache + * Close and reopens the database to read the size to a key from file again. + */ +START_TEST(test_GetDataSize) +{ + char sysTimeBuffer[256]; + int size = 0, ret = 0; + int handle = 0; + struct tm* locTime; + + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 128, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + //Cleaning up testdata folder + remove("/tmp/size-localdb.db"); + + handle = persComDbOpen("/tmp/size-localdb.db", 0x1); //create localdb.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + //write key to cache + ret = persComDbWriteKey(handle, "status/open_document", (char*) sysTimeBuffer, strlen(sysTimeBuffer)); + fail_unless(ret == strlen(sysTimeBuffer), "Wrong write size"); + +#if 1 + //get keysize from cache + size = persComDbGetKeySize(handle, "status/open_document"); + //printf("=>=>=>=> soll: %d | ist: %d\n", strlen(sysTimeBuffer), size); + fail_unless(size == strlen(sysTimeBuffer), "Invalid size read from cache"); + + //persist cached data + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + handle = persComDbOpen("/tmp/size-localdb.db", 0x1); //create localdb.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + //get keysize from file + size = persComDbGetKeySize(handle, "status/open_document"); + //printf("=>=>=>=> soll: %d | ist: %d\n", strlen(sysTimeBuffer), size); + fail_unless(size == strlen(sysTimeBuffer), "Invalid size read from file"); + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database: retval: [%d]", ret); + +#endif +} +END_TEST + +/* + * Delete a key from local DB using the key value interface. + * First write some keys, then read the keys. After that the keys get deleted. + * A further read is performed and must fail. + * The keys are inserted again and get persisted to file. the database gets reopened and the keys are deleted again. + * The next read must fail again. + */ +START_TEST(test_DeleteDataLocalDB) +{ + int rval = 0; + int handle = 0; + unsigned char buffer[READ_SIZE] = { 0 }; + char write1[READ_SIZE] = { 0 }; + char sysTimeBuffer[256]; + struct tm* locTime; + + char write2[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + +#if 1 + + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 128, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + //Cleaning up testdata folder + remove("/tmp/delete-localdb.db"); + + handle = persComDbOpen("/tmp/delete-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", handle); + + snprintf(write1, 128, "%s %s", "/70", sysTimeBuffer); + + //write to cache + int i = 0; + for(i=0; i< 300; i++) + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + rval = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(rval == strlen(write2) , "Wrong write size while inserting in cache"); + } + + //read from cache must work + for(i=0; i< 300; i++) + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + rval = persComDbReadKey(handle, key, (char*) buffer, strlen(write2)); + fail_unless(rval == strlen(write2), "Wrong read size while reading from cache"); + } + + // mark some data in cache as deleted + for(i=0; i < 6; i++) //key0 - key5 + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + rval = persComDbDeleteKey(handle, key); + fail_unless(rval >= 0, "Failed to delete key: %s", key); + } + + // after deleting the keys in cache, reading from key0 - key5 must fail now for these keys + for(i=0; i< 300; i++) + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + if(i < 6) + { + rval = persComDbReadKey(handle, key, (char*) buffer, READ_SIZE); + fail_unless(rval < 0, "Read form key [%s] works, but should fail",key); + } + else + { + rval = persComDbReadKey(handle, key, (char*) buffer, strlen(write2)); + fail_unless(rval == strlen(write2), "Wrong read size while reading from cache"); + } + } + + //persist data to file (Dlt output must show error for writeback of deleted keys (not found because they do not exist in file yet) + rval = persComDbClose(handle); + if (rval != 0) + { + printf("persComDbClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close cached database: retval: [%d]", rval); + + //open database again and write keys to cache that get persisted to file afterwards + handle = persComDbOpen("/tmp/delete-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", handle); + + + //write data again which gets persisted to file afterwards + //write data to cache + for(i=0; i< 300; i++) + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + rval = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(rval == strlen(write2) , "Wrong write size while inserting in cache"); + } + + // read data from cache must work + for(i=0; i< 300; i++) + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + rval = persComDbReadKey(handle, key, (char*) buffer, strlen(write2)); + fail_unless(rval == strlen(write2), "Wrong read size while reading from cache"); + } + + //persist data to file + rval = persComDbClose(handle); + if (rval != 0) + { + printf("persComDbClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close cached database: retval: [%d]", rval); + + //reopen database and read persisted keys + handle = persComDbOpen("/tmp/delete-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to open lDB: retval: [%d]", handle); + + // read data from file must work + for(i=0; i< 300; i++) + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + rval = persComDbReadKey(handle, key, (char*) buffer, strlen(write2)); + fail_unless(rval == strlen(write2), "Wrong read size while reading from file for key: %s", key); + } + + //delete keys (request to delete gets stored in cache) + // mark some data in cache as deleted + for(i=0; i < 6; i++) //key0 - key5 + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + rval = persComDbDeleteKey(handle, key); + fail_unless(rval >= 0, "Failed to delete key: %s", key); + } + + // after deleting the keys in cache, reading from key0 - key5 must fail now for these keys + for(i=0; i< 300; i++) + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + if(i < 6) + { + rval = persComDbReadKey(handle, key, (char*) buffer, READ_SIZE); + fail_unless(rval < 0, "Read key [%s] from cache works, but should fail",key); + } + else + { + rval = persComDbReadKey(handle, key, (char*) buffer, strlen(write2)); + fail_unless(rval == strlen(write2), "Wrong read size while reading from cache for key: %s", key); + } + } + + + //delete keys in writeback at close + rval = persComDbClose(handle); + if (rval != 0) + { + printf("persComDbClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close database: retval: [%d]", rval); + + //reopen database and try to read the deleted keys (must fail now) + handle = persComDbOpen("/tmp/delete-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to open lDB: retval: [%d]", handle); + + + // after deleting the keys in cache, reading from key0 - key5 must fail now for these keys + for(i=0; i< 300; i++) + { + snprintf(key, 128, "key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + if(i < 6) + { + rval = persComDbReadKey(handle, key, (char*) buffer, READ_SIZE); + fail_unless(rval < 0, "Read key [%s] from file works, but should fail",key); + } + else + { + rval = persComDbReadKey(handle, key, (char*) buffer, strlen(write2)); + fail_unless(rval == strlen(write2), "Wrong read size while reading from file"); + } + } + + + rval = persComDbClose(handle); + if (rval != 0) + { + printf("persComDbClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close database: retval: [%d]", rval); + + //reopen the database to write a key again into cache that must reuse the already deleted slot in the file when closing the database + handle = persComDbOpen("/tmp/delete-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", handle); + + // write a key again to test if slot in file is reused (CONTENT OF offset == 4 in kissdb.c) + rval = persComDbWriteKey(handle, "key1", (char*) write1, strlen(write1)); + fail_unless(rval == strlen(write1), "Wrong write size"); + + // write a key again to test if slot in file is reused (CONTENT OF offset == 4 in kissdb.c) + rval = persComDbWriteKey(handle, "key1", (char*) write1, strlen(write1)); + fail_unless(rval == strlen(write1), "Wrong write size"); + + + rval = persComDbClose(handle); + if (rval != 0) + { + printf("persComDbClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close database: retval: [%d]", rval); + + //reopen the database to read key1 (must work) + handle = persComDbOpen("/tmp/delete-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", handle); + + rval = persComDbReadKey(handle, "key1", (char*) buffer, READ_SIZE); + fail_unless(rval == strlen(write1), "Wrong read size for key1"); + + + rval = persComDbClose(handle); + if (rval != 0) + { + printf("persComDbClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close last database: retval: [%d]", rval); + + //Cleaning up testdata folder + remove("/tmp/delete-localdb.db"); + + //Open a new empty database and try to delete a non existent key + handle = persComDbOpen("/tmp/delete-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", handle); + + //try to delete non existent keys + for(i=0; i < 6; i++) //key0 - key5 + { + snprintf(key, 128, "non-existent-key%d",i); + snprintf(write2, 128, "DATA-%d",i ); + rval = persComDbDeleteKey(handle, key); + fail_unless(rval < 0, "Deleting key %s works, but should fail!", key); + } + + rval = persComDbClose(handle); + if (rval != 0) + { + printf("persComDbClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close last database: retval: [%d]", rval); + +#endif +} +END_TEST + + +/* + * Delete a key from local DB using the key value interface. + * First read a from a key, the delte the key + * and then try to read again. The Last read must fail. + */ +START_TEST(test_DeleteDataRct) +{ + int rval = 0; + int handle = 0; + char sysTimeBuffer[256]; + struct tm* locTime; + PersistenceConfigurationKey_s psConfig, psConfig_out; + psConfig.policy = PersistencePolicy_wt; + psConfig.storage = PersistenceStorage_local; + psConfig.type = PersistenceResourceType_key; + psConfig.permission = PersistencePermission_ReadWrite; + +#if 1 + + time_t t = time(0); + locTime = localtime(&t); + + // write data + snprintf(sysTimeBuffer, 128, "\"%s %d.%d.%d - %d:%.2d:%.2d Uhr\"", dayOfWeek[locTime->tm_wday], locTime->tm_mday, + locTime->tm_mon, (locTime->tm_year + 1900), locTime->tm_hour, locTime->tm_min, locTime->tm_sec); + + //Cleaning up testdata folder + remove("/tmp/delete-rct.db"); + + handle = persComRctOpen("/tmp/delete-rct.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent RCT: retval: [%d]", handle); + + memset(psConfig.custom_name, 0, sizeof(psConfig.custom_name)); + memset(psConfig.customID, 0, sizeof(psConfig.customID)); + memset(psConfig.reponsible, 0, sizeof(psConfig.reponsible)); + + psConfig.max_size = 12345; + char custom_name[PERS_RCT_MAX_LENGTH_CUSTOM_NAME] = "this is the custom name"; + char custom_ID[PERS_RCT_MAX_LENGTH_CUSTOM_ID] = "this is the custom ID"; + char responsible[PERS_RCT_MAX_LENGTH_RESPONSIBLE] = "this is the responsible"; + + strncpy(psConfig.custom_name, custom_name, strlen(custom_name)); + strncpy(psConfig.customID, custom_ID, strlen(custom_ID)); + strncpy(psConfig.reponsible, responsible, strlen(responsible)); + + //write key to cache + rval = persComRctWrite(handle, "key_to_delete", &psConfig); + fail_unless(rval == sizeof(psConfig), "Wrong write size in cache"); + + memset(psConfig_out.custom_name, 0, sizeof(psConfig_out.custom_name)); + memset(psConfig_out.customID, 0, sizeof(psConfig_out.customID)); + memset(psConfig_out.reponsible, 0, sizeof(psConfig_out.reponsible)); + + //read from cache + rval = persComRctRead(handle, "key_to_delete", &psConfig_out); + fail_unless(rval == sizeof(psConfig), "Wrong read size from cache"); + + +// printf("Custom ID : %s\n", psConfig_out.customID ); +// printf("Custom Name : %s\n", psConfig_out.custom_name ); +// printf("reponsible : %s\n", psConfig_out.reponsible ); +// printf("max_size : %d\n", psConfig_out.max_size ); +// printf("permission : %d\n", psConfig_out.permission ); +// printf("type : %d\n", psConfig_out.type ); +// printf("storage : %d\n", psConfig_out.storage ); +// printf("policy : %d\n", psConfig_out.policy ); + + + fail_unless(strncmp(psConfig.customID, psConfig_out.customID, strlen(psConfig_out.customID)) == 0, + "Buffer not correctly read"); + fail_unless(strncmp(psConfig.custom_name, psConfig_out.custom_name, strlen(psConfig_out.custom_name)) == 0, + "Buffer not correctly read"); + fail_unless(strncmp(psConfig.reponsible, psConfig_out.reponsible, strlen(psConfig_out.reponsible)) == 0, + "Buffer not correctly read"); + fail_unless(psConfig.max_size == psConfig_out.max_size, "Buffer not correctly read"); + fail_unless(psConfig.permission == psConfig_out.permission, "Buffer not correctly read"); + fail_unless(psConfig.policy == psConfig_out.policy, "Buffer not correctly read"); + fail_unless(psConfig.storage == psConfig_out.storage, "Buffer not correctly read"); + fail_unless(psConfig.type == psConfig_out.type, "Buffer not correctly read"); + + // mark key in cache as deleted + rval = persComRctDelete(handle, "key_to_delete"); + fail_unless(rval >= 0, "Failed to delete key"); + + // after deleting the key, reading from key in cache must fail now! + rval = persComRctRead(handle, "key_to_delete", &psConfig_out); + fail_unless(rval < 0, "Read form key [key_to_delete] works, but should fail"); + + //write data again to cache + //write key to cache which already exists in database file + rval = persComRctWrite(handle, "key_to_delete", &psConfig); + fail_unless(rval == sizeof(psConfig), "Wrong write size in cache"); + + //read from cache must work + rval = persComRctRead(handle, "key_to_delete", &psConfig_out); + fail_unless(rval == sizeof(psConfig), "Wrong read size from cache"); + + + rval = persComRctClose(handle); + if (rval != 0) + { + printf("persComRctClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close database: retval: [%d]", rval); + + handle = persComRctOpen("/tmp/delete-rct.db", 0x1); //create db if not present + fail_unless(handle >= 0, "Failed to create non existent RCT: retval: [%d]", handle); + + // mark key in cache as deleted + rval = persComRctDelete(handle, "key_to_delete"); + fail_unless(rval >= 0, "Failed to delete key"); + + // after deleting the key, reading from key must fail now! + rval = persComRctRead(handle, "key_to_delete", &psConfig_out); + fail_unless(rval < 0, "Read form key [key_to_delete] works, but should fail"); + + // try to delete a non existent key (must fail) + rval = persComRctDelete(handle, "key_to_delete_not_present"); + fail_unless(rval < 0, "Deleting key [key_to_delete_not_present] works, but should fail!"); + + // insert this key in cache + rval = persComRctWrite(handle, "key_to_delete_not_present", &psConfig); + fail_unless(rval == sizeof(psConfig), "Wrong write size in cache"); + + //read from cache must work + rval = persComRctRead(handle, "key_to_delete_not_present", &psConfig_out); + fail_unless(rval == sizeof(psConfig), "Wrong read size from cache"); + + // try to delete a key in cache that is not present in file + rval = persComRctDelete(handle, "key_to_delete_not_present"); + fail_unless(rval >= 0, "Deleting key from cache [key_to_delete_not_present] failed!"); + + //persist data to file (Dlt output must show error for writeback of deleted keys (not found because they do not exist in file yet) + rval = persComRctClose(handle); + if (rval != 0) + { + printf("persComRctClose() failed: [%d] \n", rval); + } + fail_unless(rval == 0, "Failed to close database: retval: [%d]", rval); + + +#endif +} +END_TEST + + + +/* + * + * + */ +START_TEST(test_CachedConcurrentAccess) +{ + int pid; + + //Cleaning up testdata folder + remove("/tmp/cached-concurrent.db"); + + pid = fork(); + if (pid == 0) + { + DLT_REGISTER_APP("PCOt", "tests the persistence common object library"); + + /*child*/ + //printf("Started child process with PID: [%d] \n", pid); + int handle = 0; + int ret = 0; + char childSysTimeBuffer[256] = { 0 }; + char key[128] = { 0 }; + char write2[READ_SIZE] = { 0 }; + int i =0; + + snprintf(childSysTimeBuffer, 256, "%s", "1"); + + //wait so that father has already opened the db + sleep(3); + + //open db after father (in order to use the hashtable in shared memory) + handle = persComDbOpen("/tmp/cached-concurrent.db", 0x0); //open existing test.db if not present + fail_unless(handle >= 0, "Child failed to create non existent lDB: retval: [%d]", ret); + + //read the new key written by the father from cache + for(i=0; i< 200; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i ); + ret = persComDbReadKey(handle, key, (char*) childSysTimeBuffer, 256); + //printf("Child Key Read from Cache: %s ------: %s -------------- returnvalue: %d\n",key, childSysTimeBuffer, ret); + fail_unless(ret == strlen(write2), "Child: Wrong read size"); + } + + //write to test if cache can be accessed + ret = persComDbWriteKey(handle, "write-test-concurrent", "123456", 6); + //printf("Father persComDbWriteKey: %s -- returnvalue: %d \n", key, ret); + fail_unless(ret == 6 , "CHILD: Wrong write size returned: %d", ret); + + //close database for child instance + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Child failed to close database: retval: [%d]", ret); + + DLT_UNREGISTER_APP(); + _exit(EXIT_SUCCESS); + } + else if (pid > 0) + { + /*parent*/ + //printf("Started father process with PID: [%d] \n", pid); + int handle = 0; + int ret = 0; + char write2[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + char sysTimeBuffer[256] = { 0 }; + int i =0; + + handle = persComDbOpen("/tmp/cached-concurrent.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Father failed to create non existent lDB: retval: [%d]", ret); + + //Write data to cache (cache gets created here) + for(i=0; i< 200; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + //printf("Father persComDbWriteKey: %s -- returnvalue: %d \n", key, ret); + fail_unless(ret == strlen(write2), "Father: Wrong write size"); + } + //read data from cache + for(i=0; i< 200; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i ); + ret = persComDbReadKey(handle, key, (char*) sysTimeBuffer, 256); + //printf("Father Key Read from key: %s ------: %s -------------- returnvalue: %d\n",key, sysTimeBuffer, ret); + fail_unless(ret == strlen(write2), "Father: Wrong read size"); + } + + printf("INFO: Waiting for child process to exit ... \n"); + + //wait for child exiting + int status; + (void) waitpid(pid, &status, 0); + + + //TEST if addresses have changed after child has written his data + ret = persComDbWriteKey(handle, "write-test-FATHER", "123456", 6); + //test + + + //close database for father instance (closes the cache) + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Father failed to close database: retval: [%d]", ret); + + _exit(EXIT_SUCCESS); + } +} +END_TEST + + + + +/* + * + * + */ +START_TEST(test_CachedConcurrentAccess2) +{ + int pid; + + //Cleaning up testdata folder + remove("/tmp/cached-concurrent2.db"); + + pid = fork(); + if (pid == 0) + { + DLT_REGISTER_APP("PCOt", "tests the persistence common object library"); + + /*child*/ + printf("Started child process with PID: [%d] \n", pid); + int handle = 0; + int ret = 0; + char childSysTimeBuffer[256] = { 0 }; + char key[128] = { 0 }; + char write2[READ_SIZE] = { 0 }; + int i =0; + + snprintf(childSysTimeBuffer, 256, "%s", "1"); + + //open database initially (CREATOR) + handle = persComDbOpen("/tmp/cached-concurrent2.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Child failed to create non existent lDB: retval: [%d]", ret); + + + for(i=0; i< 200; i++) + { + snprintf(key, 128, "CHILD_Key_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + //printf("Father persComDbWriteKey: %s -- returnvalue: %d \n", key, ret); + fail_unless(ret == strlen(write2), "Child: Wrong write size"); + } + + //read the new key written by the father from cache +// for(i=0; i< 200; i++) +// { +// snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); +// snprintf(write2, 128, "DATA-%d",i ); +// ret = persComDbReadKey(handle, key, (char*) childSysTimeBuffer, 256); +// //printf("Child Key Read from Cache: %s ------: %s -------------- returnvalue: %d\n",key, childSysTimeBuffer, ret); +// fail_unless(ret == strlen(write2), "Child: Wrong read size"); +// } + + //write in order to create cache + ret = persComDbWriteKey(handle, "write-test-concurrent-CHILD", "123456", 6); + printf("Child persComDbWriteKey: write-test-concurrent-CHILD -- returnvalue: %d \n", ret); + fail_unless(ret == 6 , "CHILD: Wrong write size returned: %d", ret); + + //wait until child has also accessed the cache + sleep(2); + + //close database for child instance (CREATOR but not the last instance using the database) + printf("Child (Creator) closes database! \n"); + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Child failed to close database: retval: [%d]", ret); + + DLT_UNREGISTER_APP(); + _exit(EXIT_SUCCESS); + } + else if (pid > 0) + { + /*parent*/ + printf("Started father process with PID: [%d] \n", pid); + int handle = 0; + int ret = 0; + char write2[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + char sysTimeBuffer[256] = { 0 }; + int i =0; + + //wait until child (CREATOR) has opened the database + sleep(1); + + handle = persComDbOpen("/tmp/cached-concurrent2.db", 0x0); //open existing database + fail_unless(handle >= 0, "Father failed to create non existent lDB: retval: [%d]", ret); + + + //Write data to already created cache + for(i=0; i< 200; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + //printf("Father persComDbWriteKey: %s -- returnvalue: %d \n", key, ret); + fail_unless(ret == strlen(write2), "Father: Wrong write size"); + } + //read data from cache + for(i=0; i< 200; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i ); + ret = persComDbReadKey(handle, key, (char*) sysTimeBuffer, 256); + //printf("Father Key Read from key: %s ------: %s -------------- returnvalue: %d\n",key, sysTimeBuffer, ret); + fail_unless(ret == strlen(write2), "Father: Wrong read size"); + } + + printf("INFO: Waiting for child process to exit ... \n"); + + //wait for child exiting + int status; + (void) waitpid(pid, &status, 0); + + + //TEST if addresses have changed after child has written his data + ret = persComDbWriteKey(handle, "write-test-FATHER", "123456", 6); + //test + + + //close database for father instance that has NOT created the database and cache (last instance that must do the writeback) + printf("Father (not the Creator) closes database! \n"); + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Father failed to close database: retval: [%d]", ret); + + _exit(EXIT_SUCCESS); + } +} +END_TEST + + + + +START_TEST(test_CacheSize) +{ + unsigned char buffer2[PERS_DB_MAX_SIZE_KEY_DATA] = { 1 }; + int handle = 0; + int i, k, ret = 0; + int maxKeys = 2169; + char dataBufer[PERS_DB_MAX_SIZE_KEY_DATA] = { 1 }; + char key[128] = { 0 }; + char path[128] = { 0 }; + int handles[100] = { 0 }; + int writings = 2173; + int databases = 1; + + for (k = 0; k < databases; k++) + { + snprintf(path, 128, "/tmp/cacheSize-%d.db", k); + //Cleaning up testdata folder + remove(path); + } + + //use maximum allowed data size + for (i = 0; i < PERS_DB_MAX_SIZE_KEY_DATA; i++) + { + dataBufer[i] = 'x'; + } + dataBufer[PERS_DB_MAX_SIZE_KEY_DATA-1] = '\0'; + + //fill k databases with i key /value pairs + for (k = 0; k < databases; k++) + { + snprintf(path, 128, "/tmp/cacheSize-%d.db", k); + handle = persComDbOpen(path, 0x1); //create test.db if not present + handles[k] = handle; + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + //write data to cache + for (i = 0; i < writings; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d", i, i * i); + ret = persComDbWriteKey(handle, key, (char*) dataBufer, strlen(dataBufer)); + //printf("Writing Key: %s | Retval: %d \n", key, ret); + + if( i >= maxKeys) //write must fail (adapt maxKeySize if cache size is increased or item size gets decreased) + { + fail_unless(ret < 0 , "Insert in cache works but should fail!: %d", ret); + } + else //write must work + { + fail_unless(ret == strlen(dataBufer) , "Wrong write size while inserting in cache"); + } + } + + //read data from cache + for (i = 0; i < writings; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d", i, i * i); + ret = persComDbReadKey(handle, key, (char*) buffer2, PERS_DB_MAX_SIZE_KEY_DATA); + //printf("read from key: %s | Retval: %d \n", key, ret); + + if( i >= maxKeys) //read must fail + { + fail_unless(ret < 0, "Read from cache works, but should fail!: %d", ret); + } + else //read must work + { + fail_unless(ret == strlen(dataBufer), "Wrong read size while reading from cache"); + } + } + } + + //sleep to look for memory consumption of cache + //sleep(15); + //Close k databases in order to persist the data + for (k = 0; k < databases; k++) + { + ret = persComDbClose(handles[k]); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database with number %d: retval: [%d]", k, ret); + } +} +END_TEST + + + + +START_TEST(test_BadParameters) +{ + //perscomdbopen + char* path = NULL; + int ret = 0; + + //percomDBwrite test + char buffer[16384] = { 0 }; + char* Badbuffer = NULL; + char* key = NULL; + char* data = NULL; + + //rct write + PersistenceConfigurationKey_s* BadPsConfig = NULL; + PersistenceConfigurationKey_s psConfig; + + ret = persComDbOpen("6l3HrKvT9TmXdTbqF4mc3N38llpbKkn2qMhdiLOlwXnY7H09ZewvQG80uyLr8sg0by0oD9UXNOm9OHvXl8zf7vKosu0M90Aau6WFqELDJl6OYr3xPYPH59o7AvixDMQXlNrPUUTdluU24TEFEiTVhcRcWJDoxlL6LHg1u9p3pNURI9GKmAsXDHovXXrvwP3qSjDYB0gMhvEvfpDI5oy8vb3Frz81zZmKuHsx9GQi0xWTB5n6grRH9TvcJW7F1yu7", + 0x0); //Pathname exceeds 255 chars + fail_unless(ret < 0, "Open local db with too long path length works, but should fail: retval: [%d]", ret); + + ret = persComDbOpen(path, 0x1); //create test.db if not present + fail_unless(ret < 0, "Open local db with bad pathname works, but should fail: retval: [%d]", ret); + + ret = persComDbClose(-1); + fail_unless(ret < 0, "Closing the local database with negative handle works, but should fail: retval: [%d]", ret); + + ret = persComRctOpen("6l3HrKvT9TmXdTbqF4mc3N38llpbKkn2qMhdiLOlwXnY7H09ZewvQG80uyLr8sg0by0oD9UXNOm9OHvXl8zf7vKosu0M90Aau6WFqELDJl6OYr3xPYPH59o7AvixDMQXlNrPUUTdluU24TEFEiTVhcRcWJDoxlL6LHg1u9p3pNURI9GKmAsXDHovXXrvwP3qSjDYB0gMhvEvfpDI5oy8vb3Frz81zZmKuHsx9GQi0xWTB5n6grRH9TvcJW7F1yu7", + 0x0); //Pathname exceeds 255 chars + fail_unless(ret < 0, "Open RCT db with too long path length works, but should fail: retval: [%d]", ret); + + ret = persComRctOpen(path, 0x1); //create test.db if not present + fail_unless(ret < 0, "Open RCT db with bad pathname works, but should fail: retval: [%d]", ret); + + ret = persComRctClose(-1); + fail_unless(ret < 0, "Closing the RCT database with negative handle works, but should fail: retval: [%d]", ret); + + ret = persComDbWriteKey(1, "CteKh3FTonalS4AlOaEruzUbgAP9fryYJLCykq5tTPQkPrHEcV9p6akxa6TuF9gqnJu5iCEyxMUu17QhTP7sYgFwFKU1qqNMcCmps8WcpWDR2oCnjqdaBtATL2A36q6QV", "data", 4); //key too long + fail_unless(ret < 0 , "Writing with wrong keylength works, but should fail"); + + ret = persComDbWriteKey(-1, "key", "data", 4); //negative handle + fail_unless(ret < 0 , "Writing with negative handle works, but should fail"); + + ret = persComDbWriteKey(1, key, "data", 4); //key is NULL + fail_unless(ret < 0 , "Writing key that is NULL works, but should fail"); + + ret = persComDbWriteKey(1, "key", data, 4); //data is NULL + fail_unless(ret < 0 , "Writing data that is NULL works, but should fail"); + + ret = persComDbWriteKey(1, "key", "data", -1254); //datasize is negative + fail_unless(ret < 0 , "Writing with negative datasize works, but should fail"); + + ret = persComDbWriteKey(1, "key", "data", 0); //datasize is zero + fail_unless(ret < 0 , "Writing with zero datasize works, but should fail"); + + ret = persComDbWriteKey(1, "key", "data", 16385); //datasize too big + fail_unless(ret < 0 , "Writing with too big datasize works, but should fail"); + + ret = persComRctWrite(1, "CteKh3FTonalS4AlOaEruzUbgAP9fryYJLCykq5tTPQkPrHEcV9p6akxa6TuF9gqnJu5iCEyxMUu17QhTP7sYgFwFKU1qqNMcCmps8WcpWDR2oCnjqdaBtATL2A36q6QV", &psConfig); //key too long + fail_unless(ret < 0 , "Writing RCT with wrong keylength works, but should fail"); + + ret = persComRctWrite(-1, "key", &psConfig); //negative handle + fail_unless(ret < 0 , "Writing RCT with negative handle works, but should fail"); + + ret = persComRctWrite(1, key, &psConfig); //key is NULL + fail_unless(ret < 0 , "Writing RCT key that is NULL works, but should fail"); + + ret = persComRctWrite(1, "key", BadPsConfig); //data is NULL + fail_unless(ret < 0 , "Writing RCT data that is NULL works, but should fail"); + + ret = persComDbReadKey(1, "CteKh3FTonalS4AlOaEruzUbgAP9fryYJLCykq5tTPQkPrHEcV9p6akxa6TuF9gqnJu5iCEyxMUu17QhTP7sYgFwFKU1qqNMcCmps8WcpWDR2oCnjqdaBtATL2A36q6QV", "data", 4); //key too long + fail_unless(ret < 0 , "Reading with wrong keylength works, but should fail"); + + ret = persComDbReadKey(-1, "key", buffer, 4); //negative handle + fail_unless(ret < 0 , "Reading with negative handle works, but should fail"); + + ret = persComDbReadKey(1, key, buffer, 4); //key is NULL + fail_unless(ret < 0 , "Reading key that is NULL works, but should fail"); + + ret = persComDbReadKey(1, "key", Badbuffer, 4); //data is NULL + fail_unless(ret < 0 , "Reading data to buffer that is NULL works, but should fail"); + + ret = persComDbReadKey(1, "key", buffer, -1254); //data buffer size is negative + fail_unless(ret < 0 , "Reading with negative data buffer size works, but should fail"); + + ret = persComDbReadKey(1, "key", buffer, 0); //data buffer size is zero + fail_unless(ret < 0 , "Reading with zero data buffer size works, but should fail"); + + ret = persComRctRead(1, "CteKh3FTonalS4AlOaEruzUbgAP9fryYJLCykq5tTPQkPrHEcV9p6akxa6TuF9gqnJu5iCEyxMUu17QhTP7sYgFwFKU1qqNMcCmps8WcpWDR2oCnjqdaBtATL2A36q6QV", &psConfig); //key too long + fail_unless(ret < 0 , "Reading RCT with wrong keylength works, but should fail"); + + ret = persComRctRead(-1, "key", &psConfig); //negative handle + fail_unless(ret < 0 , "Reading RCT with negative handle works, but should fail"); + + ret = persComRctRead(1, key, &psConfig); //key is NULL + fail_unless(ret < 0 , "Reading RCT key that is NULL works, but should fail"); + + ret = persComRctRead(1, "key", BadPsConfig); //data is NULL + fail_unless(ret < 0 , "Reading RCT data to buffer that is NULL works, but should fail"); + + ret = persComDbGetSizeKeysList(-1); + fail_unless(ret < 0 , "Reading keylist size with negative handle works, but should fail"); + + ret = persComRctGetSizeResourcesList(-1); + fail_unless(ret < 0 , "Reading RCT resourcelist size with negative handle works, but should fail"); + + ret = persComDbGetKeySize(1, "CteKh3FTonalS4AlOaEruzUbgAP9fryYJLCykq5tTPQkPrHEcV9p6akxa6TuF9gqnJu5iCEyxMUu17QhTP7sYgFwFKU1qqNMcCmps8WcpWDR2oCnjqdaBtATL2A36q6QV"); //key too long + fail_unless(ret < 0 , "Reading Size with wrong keylength works, but should fail"); + + ret = persComDbGetKeySize(-1, "key"); //negative handle + fail_unless(ret < 0 , "Reading Size with negative handle works, but should fail"); + + ret = persComDbGetKeySize(1, key); //key is NULL + fail_unless(ret < 0 , "Reading Size from key that is NULL works, but should fail"); + + ret = persComDbGetKeysList(-1, buffer, 10); // + fail_unless(ret < 0 , "Reading key list with negative handle works, but should fail"); + + ret = persComDbGetKeysList(1, Badbuffer, 10); // + fail_unless(ret < 0 , "Reading key list with readbuffer that is NULL works, but should fail"); + + ret = persComDbGetKeysList(1, buffer, -1); // + fail_unless(ret < 0 , "Reading key list with negative buffer size works, but should fail"); + + ret = persComDbGetKeysList(1, buffer, 0); // + fail_unless(ret < 0 , "Reading key list with zero buffer size works, but should fail"); + + ret = persComRctGetResourcesList(-1, buffer, 10); // + fail_unless(ret < 0 , "Reading RCT key list with negative handle works, but should fail"); + + ret = persComRctGetResourcesList(1, Badbuffer, 10); // + fail_unless(ret < 0 , "Reading RCT key list with readbuffer that is NULL works, but should fail"); + + ret = persComRctGetResourcesList(1, buffer, -1); // + fail_unless(ret < 0 , "Reading RCT key list with negative buffer size works, but should fail"); + + ret = persComRctGetResourcesList(1, buffer, 0); // + fail_unless(ret < 0 , "Reading RCT key list with zero buffer size works, but should fail"); + + ret = persComDbDeleteKey(1, "CteKh3FTonalS4AlOaEruzUbgAP9fryYJLCykq5tTPQkPrHEcV9p6akxa6TuF9gqnJu5iCEyxMUu17QhTP7sYgFwFKU1qqNMcCmps8WcpWDR2oCnjqdaBtATL2A36q6QV"); + fail_unless(ret < 0 , "Deleting key with wrong keylength works, but should fail"); + + ret = persComDbDeleteKey(1, key); + fail_unless(ret < 0 , "Deleting key with key that is NULL works, but should fail"); + + ret = persComDbDeleteKey(-1, "key"); + fail_unless(ret < 0 , "Deleting with negative handle works, but should fail"); + + ret = persComRctDelete(1, "CteKh3FTonalS4AlOaEruzUbgAP9fryYJLCykq5tTPQkPrHEcV9p6akxa6TuF9gqnJu5iCEyxMUu17QhTP7sYgFwFKU1qqNMcCmps8WcpWDR2oCnjqdaBtATL2A36q6QV"); + fail_unless(ret < 0 , "Deleting RCT key with wrong keylength works, but should fail"); + + ret = persComRctDelete(1, key); + fail_unless(ret < 0 , "Deleting RCT key with key that is NULL works, but should fail"); + + ret = persComRctDelete(-1, "key"); + fail_unless(ret < 0 , "Deleting RCT key with negative handle works, but should fail"); +} +END_TEST + + +/* + * This test first writes a valid database file. + * Then the hashtable area of the database file is made corrupt + * In the last step, the database is reopened again. + * If the corrupt hashtables are idenfified and resbuilt correctly, + * all keys that were written in the first step, must be readable. + */ +START_TEST(test_RebuildHashtables) +{ + int ret = 0; + int handle = 0; + char write2[READ_SIZE] = { 0 }; + char read[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + int i =0; + + //Cleaning up testdata folder + remove("/tmp/rebuild-hashtables.db"); + +#if 1 + memset(write2, 0 , sizeof(write2)); + snprintf(write2, 176 , "%s", + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/"); + + handle = persComDbOpen("/tmp/rebuild-hashtables.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + //write to cache + for(i=0; i < 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2) , "Wrong write size while inserting in cache"); + } + + //read from cache + for(i=0; i < 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + memset(read, 0 , sizeof(read)); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong read size while reading from cache"); + fail_unless(memcmp(read, write2, sizeof(write2)) == 0, "Reading Data from Cache failed: Buffer not correctly read"); + } + + //persist data in cache to file + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + //reopen to delete a key + handle = persComDbOpen("/tmp/rebuild-hashtables.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + ret = persComDbDeleteKey(handle, "Key_in_loop_2_4"); + fail_unless(ret >= 0, "Failed to delete key"); + + //persist deleted data in cache to file + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + // IF DATABASE HEADER STRUCTURES OR KEY VALUE PAIR STORAGE CHANGES, the seek to offset part must be updated + //open database and make data corrupt + int fd; + FILE* f; + fd = open("/tmp/rebuild-hashtables.db", O_RDWR , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH ); //gets closed when f is closed + f = fdopen(fd, "w+b"); + + uint64_t flag = 0x01; + + //seek to close failed flag and set it to 1 (to invoke check of hashtables) + fseeko(f,16, SEEK_SET); + fwrite(&flag,sizeof(uint64_t),1, f); + + //seek to data of hashtable area (to destroy a hashtable) + fseeko(f,4105, SEEK_SET); + fputc('x',f); //make data corrupt + + //seek to data of hashtable area + fseeko(f,6802, SEEK_SET); + fputc('x',f); //make data corrupt + + //seek to data of hashtable area + fseeko(f,15995, SEEK_SET); + fputc('x',f); //make data corrupt + + //destroy delimiters of a hashtable + // destroy start delimiter of a hashtable - 655352 + fseeko(f,655352, SEEK_SET); + fputc('x',f); + + + + + + + //just make block B data corrupt- -> block A must be used for recovery + fseeko(f,4841626, SEEK_SET); + fputc('x',f); + + //destroy one delimiter of datablock A --> Key_in_loop_222_49284 --> block A can be used for recovery if data is valid + fseeko(f,1024002, SEEK_SET); + fputc('x',f); + + //Destroy data of block A --> Key_in_loop_153_23409 --> block B must be used for recovery + fseeko(f,16407, SEEK_SET); + fputc('x',f); //just make block A data corrupt + + //destroy both delimiters of datablock A --> Key_in_loop_101_10201 --> block B must be used for recovery + fseeko(f,827394, SEEK_SET); + fputc('x',f); + fseeko(f,835577, SEEK_SET); + fputc('x',f); + + //also destroy both delimiters of last datablock A in file --> Key_in_loop_4_16 --> block B must be used for recovery + fseeko(f,4964353, SEEK_SET); + fputc('x',f); + fseeko(f,4972537, SEEK_SET); + fputc('x',f); + + //make block A and block B data corrupt --> Key_in_loop_31_961 --> recovery not possible + fseeko(f,3834005, SEEK_SET); + fputc('x',f); + fseeko(f,3842201, SEEK_SET); + fputc('x',f); + + //test with start AND end delimiter of hashtable destroyed --> recovery not possible +// fseeko(f,4098, SEEK_SET); +// fputc('y',f); +// fseeko(f,16377, SEEK_SET); +// fputc('y',f); + + fclose(f); + + //printf("reopen destroyed database\n"); + + handle = persComDbOpen("/tmp/rebuild-hashtables.db", 0x1); //reopen database with corrupted hashtables and corrupted datablocks + fail_unless(handle >= 0, "Failed to reopen existing lDB: retval: [%d]", ret); + memset(read, 0 , sizeof(read)); + //read from database file must work if rebuild of hashtables was successful + for(i=0; i < 300; i++) + { + //printf("read verification \n"); + if (i != 2 && i != 31) //do not expect successful read for deleted data or not recoverable data + { + snprintf(key, 128, "Key_in_loop_%d_%d", i, i * i); //Key_in_loop_0_0 + memset(read, 0, sizeof(read)); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + //printf("read key: %s returnval: %d Data: %s \n",key, ret, read); + memset(read, 0, sizeof(read)); + fail_unless(ret == strlen(write2), "Wrong read size returned for key: %s \n", key); + } + else //expect read fail for unrecoverable data ( key_31) and for deleted data( + { + snprintf(key, 128, "Key_in_loop_%d_%d", i, i * i); //Key_in_loop_0_0 + memset(read, 0, sizeof(read)); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + memset(read, 0, sizeof(read)); + fail_unless(ret < 0 , "Key: <%s> could be read, but should not be readable!\n", key); + } + } + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +#endif +} +END_TEST + + + +/* + * In this test, the recovery of corrupted datablocks is tested. + * First, a a valid database file gets written. + * Then some datablocks in the data area of the database file is made corrupt. + * In the last step, the database is reopened again. + * If the corrupt data gets idenfified and recovered correctly, + * all key value pairs that were written in the first step, must be readable. + */ +START_TEST(test_RecoverDatablocks) +{ + int ret = 0; + int handle = 0; + char write2[READ_SIZE] = { 0 }; + char read[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + int i =0; + + //Cleaning up testdata folder + remove("/tmp/recover-datablocks.db"); + +#if 1 + memset(write2, 0 , sizeof(write2)); + snprintf(write2, 176 , "%s", + "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/"); + + + handle = persComDbOpen("/tmp/recover-datablocks.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + + //write to cache + for(i=0; i < 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2) , "Wrong write size while inserting in cache"); + } + + //read from cache + for(i=0; i < 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + memset(read, 0 , sizeof(read)); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong read size while reading from cache"); + fail_unless(memcmp(read, write2, sizeof(write2)) == 0, "Reading Data from Cache failed: Buffer not correctly read"); + } + + //persist data in cache to file + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + + //printf("Database created, now destroying datablocks.....\n"); + + // IF DATABASE HEADER STRUCTURES OR KEY VALUE PAIR STORAGE CHANGES, the seek to offset part must be updated + //open database and make data corrupt + int fd; + FILE* f; + fd = open("/tmp/recover-datablocks.db", O_RDWR , S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH ); //gets closed when f is closed + f = fdopen(fd, "w+b"); + uint64_t flag = 0x01; + + //seek to close failed flag and set it to 1 + fseeko(f,16, SEEK_SET); + fwrite(&flag,sizeof(uint64_t),1, f); + + //seek to data block A of key Key_in_loop_153_23409 + fseeko(f,16407, SEEK_SET); + fputc('x',f); //make key corrupt + + //seek to data block B of key: Key_in_loop_285_81225 + fseeko(f,356559, SEEK_SET); + fputc('x',f); //make data corrupt + + //seek to data block B of key: Key_in_loop_125_15625 + fseeko(f,1212614, SEEK_SET); + fputc('x',f); //make data corrupt + + //make both blocks corrupt of key: Key_in_loop_48_2304 --> DLT_LOG must show -> datablock recovery impossible -> both datablocks are invalid! + + //block A Key_in_loop_48_2304 + fseeko(f,51066, SEEK_SET); + fputc('x',f); //make data corrupt + + //block B Key_in_loop_48_2304 + fseeko(f,61009, SEEK_SET); + fputc('x',f); //make data corrupt + + fclose(f); + + handle = persComDbOpen("/tmp/recover-datablocks.db", 0x1); //reopen database with corrupted hashtables + fail_unless(handle >= 0, "Failed to reopen existing lDB: retval: [%d]", ret); + memset(read, 0 , sizeof(read)); + //read from database file must work if rebuild of hashtables was successful + + //printf("opening database finished, start reading to verify...\n"); + for(i=0; i < 300; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); //Key_in_loop_0_0 + memset(read, 0 , sizeof(read)); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write2)); + //printf("read for key: %s returnval: %d Data: %s \n", key, ret, read); + if(i == 48) + { + //printf("48: --> ret: %d \n",ret); //ret must be negative -> key 48 and 41 have same hash value + fail_unless(ret != strlen(write2), "Read was possible but should not because both datablocks are corrupt!"); + } + else + { + memset(read, 0 , sizeof(read)); + fail_unless(ret == strlen(write2), "Wrong read size"); + } + } + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +#endif +} +END_TEST + + + + + +/* + * In this test, the access to databases through symlinks is tested + * the symlink named "/tmp/symlink" points to the folder "/tmp" + * The database file gets created under the path: "/tmp/symlink/symlink-localdb.db" + * Keys get written and must also be readable again if the database under the symlink is reopened. + */ +START_TEST(test_LinkedDatabase) +{ + int ret = 0; + int handle = 0; + unsigned char readBuffer[READ_SIZE] = { 0 }; + char write2[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + int i = 0; + + //Cleaning up testdata folder + remove("/tmp/symlink-localdb.db"); + remove("/tmp/symlink"); + + ret = symlink("/tmp","/tmp/symlink"); + fail_unless(ret == 0, "Failed to create symlink /tmp/symlink: [%s]", strerror(errno)); + + handle = persComDbOpen("/tmp/symlink/symlink-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + //write keys to cache + for(i=0; i< 50; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + memset(write2, 0, sizeof(write2)); + snprintf(write2, 128, "DATA-%d-%d",i,i*i ); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2), "Wrong write size"); + } + + //Read keys from cache + for(i=0; i< 50; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + memset(write2, 0, sizeof(write2)); + snprintf(write2, sizeof(write2), "DATA-%d-%d",i,i*i ); + memset(readBuffer, 0, sizeof(readBuffer)); + ret = persComDbReadKey(handle, key, (char*) readBuffer, sizeof(readBuffer)); + fail_unless(ret == strlen(write2), "Wrong read size"); + fail_unless(memcmp(readBuffer, write2, sizeof(readBuffer)) == 0, "Reading Data from Cache failed: Buffer not correctly read"); + } + + //persist changed data for this lifecycle + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: Cached Data was not written back: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + //open database again + handle = persComDbOpen("/tmp/symlink/symlink-localdb.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to reopen existing lDB: retval: [%d]", ret); + + //Read keys from database file + for(i=0; i< 50; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + memset(write2, 0, sizeof(write2)); + snprintf(write2, sizeof(write2), "DATA-%d-%d",i,i*i ); + memset(readBuffer, 0, sizeof(readBuffer)); + ret = persComDbReadKey(handle, key, (char*) readBuffer, sizeof(readBuffer)); + fail_unless(ret == strlen(write2), "Wrong read size"); + fail_unless(memcmp(readBuffer, write2, sizeof(readBuffer)) == 0, "Reading Data from File failed: Buffer not correctly read"); + } + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +} +END_TEST + + + + +/* + * Write a key value pair into the cache multiple times with same key but different data + * Then close the database in order to persist the data. + * After reopening the database, the last data written to cache must be returned for the key + */ +START_TEST(test_MultipleWrites) +{ + int ret = 0; + int handle = 0; + char write1[READ_SIZE] = { 0 }; + char write2[READ_SIZE] = { 0 }; + char read[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + int i =0; + + //Cleaning up testdata folder + remove("/tmp/multiple-writes.db"); + +#if 1 + + handle = persComDbOpen("/tmp/multiple-writes.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to create non existent lDB: retval: [%d]", ret); + + //use same key for all writes and reads + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write1, 128, "%s", "1234567890"); + snprintf(write2, 128, "%s", "12345"); + + //write 10 bytes to cache + ret = persComDbWriteKey(handle, key, (char*) write1, strlen(write1)); + fail_unless(ret == strlen(write1) , "Wrong write size while inserting in cache"); + //write 5 bytes to cache + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + fail_unless(ret == strlen(write2) , "Wrong write size while inserting in cache"); + + //read must return 5 bytes from cache + ret = persComDbReadKey(handle, key, (char*) read, READ_SIZE); + //printf("Read from cache write 2: %s \n", read); + fail_unless(ret == strlen(write2), "Wrong read size while reading from cache!"); + fail_unless(memcmp(read, write2, sizeof(write2)) == 0, "Reading Data from Cache failed: Buffer not correctly read!"); + + //write 10 bytes to cache + ret = persComDbWriteKey(handle, key, (char*) write1, strlen(write1)); + fail_unless(ret == strlen(write1) , "Wrong write size while inserting in cache"); + + //read must return 10 bytes from cache + ret = persComDbReadKey(handle, key, (char*) read, READ_SIZE); + //printf("Read from cache write 1: %s \n", read); + fail_unless(ret == strlen(write1), "Wrong read size while reading from cache!"); + fail_unless(memcmp(read, write1, sizeof(write1)) == 0, "Reading Data from Cache failed: Buffer not correctly read!"); + + + //persist data in cache to file + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close cached database: retval: [%d]", ret); + + + handle = persComDbOpen("/tmp/multiple-writes.db", 0x1); //create test.db if not present + fail_unless(handle >= 0, "Failed to reopen existing lDB: retval: [%d]", ret); + //printf("open ok \n"); + + //read from database file + memset(read, 0, 1024); + ret = persComDbReadKey(handle, key, (char*) read, strlen(write1)); + //printf("read: %d returns: %s \n", i, read); + fail_unless(ret == strlen(write1), "Wrong read size"); + fail_unless(memcmp(read, write1, sizeof(write1)) == 0, "Reading Data from File failed: Buffer not correctly read"); + + + + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Failed to close database file: retval: [%d]", ret); + +#endif +} +END_TEST + + + +START_TEST(test_WriteThrough) +{ + int pid; + + //Cleaning up testdata folder + remove("/tmp/writethrough-concurrent.db"); + + pid = fork(); + if (pid == 0) + { + DLT_REGISTER_APP("PCOt", "tests the persistence common object library"); + + /*child*/ + //printf("Started child process with PID: [%d] \n", pid); + int handle = 0; + int ret = 0; + char childSysTimeBuffer[256] = { 0 }; + char key[128] = { 0 }; + char write2[READ_SIZE] = { 0 }; + int i =0; + + snprintf(childSysTimeBuffer, 256, "%s", "1"); + + //wait so that father has already opened the db + //sleep(3); + + //open db after father (in order to use the hashtable in shared memory) + handle = persComDbOpen("/tmp/writethrough-concurrent.db", 0x3); //create test.db if not present and writethrough mode + fail_unless(handle >= 0, "Child failed to create non existent lDB: retval: [%d]", ret); + + //read the new key written by the father from cache + for(i=0; i< 200; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i ); + ret = persComDbReadKey(handle, key, (char*) childSysTimeBuffer, 256); + //printf("Child Key Read from Cache: %s ------: %s -------------- returnvalue: %d\n",key, childSysTimeBuffer, ret); + //fail_unless(ret == strlen(write2), "Child: Wrong read size"); + } + + //write to test if cache can be accessed + ret = persComDbWriteKey(handle, "writethrough-test-concurrent", "123456", 6); + //printf("Father persComDbWriteKey: %s -- returnvalue: %d \n", key, ret); + fail_unless(ret == 6 , "CHILD: Wrong write size returned: %d", ret); + + //close database for child instance + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Child failed to close database: retval: [%d]", ret); + + DLT_UNREGISTER_APP(); + _exit(EXIT_SUCCESS); + } + else if (pid > 0) + { + /*parent*/ + //printf("Started father process with PID: [%d] \n", pid); + int handle = 0; + int ret = 0; + char write2[READ_SIZE] = { 0 }; + char key[128] = { 0 }; + char sysTimeBuffer[256] = { 0 }; + int i =0; + + handle = persComDbOpen("/tmp/writethrough-concurrent.db", 0x3); //create test.db if not present and writethrough mode + fail_unless(handle >= 0, "Father failed to create non existent lDB: retval: [%d]", ret); + + //Write data to cache (cache gets created here) + for(i=0; i< 200; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i); + ret = persComDbWriteKey(handle, key, (char*) write2, strlen(write2)); + //printf("Father persComDbWriteKey: %s -- returnvalue: %d \n", key, ret); + fail_unless(ret == strlen(write2), "Father: Wrong write size"); + } + //read data from cache + for(i=0; i< 200; i++) + { + snprintf(key, 128, "Key_in_loop_%d_%d",i,i*i); + snprintf(write2, 128, "DATA-%d",i ); + ret = persComDbReadKey(handle, key, (char*) sysTimeBuffer, 256); + //printf("Father Key Read from key: %s ------: %s -------------- returnvalue: %d\n",key, sysTimeBuffer, ret); + //fail_unless(ret == strlen(write2), "Father: Wrong read size"); + } + + printf("INFO: Waiting for child process to exit ... \n"); + + //wait for child exiting + int status; + (void) waitpid(pid, &status, 0); + + + //TEST if addresses have changed after child has written his data + ret = persComDbWriteKey(handle, "write-test-FATHER", "123456", 6); + //test + + + //close database for father instance (closes the cache) + ret = persComDbClose(handle); + if (ret != 0) + { + printf("persComDbClose() failed: [%d] \n", ret); + } + fail_unless(ret == 0, "Father failed to close database: retval: [%d]", ret); + + _exit(EXIT_SUCCESS); + } + + +} +END_TEST + + + +static Suite* persistenceCommonLib_suite() +{ + Suite* s = suite_create("Persistence-common-object-test"); + + TCase* tc_persOpenLocalDB = tcase_create("OpenlocalDB"); + tcase_add_test(tc_persOpenLocalDB, test_OpenLocalDB); + + TCase* tc_persOpenRCT = tcase_create("OpenRCT"); + tcase_add_test(tc_persOpenRCT, test_OpenRCT); + + TCase* tc_persSetDataLocalDB = tcase_create("SetDataLocalDB"); + tcase_add_test(tc_persSetDataLocalDB, test_SetDataLocalDB); + + TCase* tc_persGetDataLocalDB = tcase_create("GetDataLocalDB"); + tcase_add_test(tc_persGetDataLocalDB, test_GetDataLocalDB); + + TCase* tc_persSetDataRCT = tcase_create("SetDataRCT"); + tcase_add_test(tc_persSetDataRCT, test_SetDataRCT); + + TCase* tc_persGetDataRCT = tcase_create("GetDataRCT"); + tcase_add_test(tc_persGetDataRCT, test_GetDataRCT); + + TCase* tc_persGetDataSize = tcase_create("GetDataSize"); + tcase_add_test(tc_persGetDataSize, test_GetDataSize); + + TCase* tc_persDeleteDataLocalDB = tcase_create("DeleteDataLocalDB"); + tcase_add_test(tc_persDeleteDataLocalDB, test_DeleteDataLocalDB); + + TCase* tc_persDeleteDataRct = tcase_create("DeleteDataRct"); + tcase_add_test(tc_persDeleteDataRct, test_DeleteDataRct); + + TCase* tc_persGetKeyListSizeLocalDB = tcase_create("GetKeyListSizeLocalDb"); + tcase_add_test(tc_persGetKeyListSizeLocalDB, test_GetKeyListSizeLocalDB); + + TCase* tc_persGetKeyListLocalDB = tcase_create("GetKeyListLocalDb"); + tcase_add_test(tc_persGetKeyListLocalDB, test_GetKeyListLocalDB); + + TCase* tc_persGetResourceListSizeRct = tcase_create("GetResourceListSizeRct"); + tcase_add_test(tc_persGetResourceListSizeRct, test_GetResourceListSizeRct); + + TCase* tc_persGetResourceListRct = tcase_create("GetResourceListRct"); + tcase_add_test(tc_persGetResourceListRct, test_GetResourceListRct); + + TCase* tc_persCacheSize = tcase_create("CacheSize"); + tcase_add_test(tc_persCacheSize, test_CacheSize); + tcase_set_timeout(tc_persCacheSize, 20); + + TCase* tc_persCachedConcurrentAccess = tcase_create("CachedConcurrentAccess"); + tcase_add_test(tc_persCachedConcurrentAccess, test_CachedConcurrentAccess); + tcase_set_timeout(tc_persCachedConcurrentAccess, 20); + + TCase* tc_persCachedConcurrentAccess2 = tcase_create("CachedConcurrentAccess2"); + tcase_add_test(tc_persCachedConcurrentAccess2, test_CachedConcurrentAccess2); + tcase_set_timeout(tc_persCachedConcurrentAccess2, 20); + + TCase* tc_BadParameters = tcase_create("BadParameters"); + tcase_add_test(tc_BadParameters, test_BadParameters); + + TCase* tc_RebuildHashtables = tcase_create("RebuildHashtables"); + tcase_add_test(tc_RebuildHashtables, test_RebuildHashtables); + + TCase* tc_RecoverDatablocks = tcase_create("RecoverDatablocks"); + tcase_add_test(tc_RecoverDatablocks, test_RecoverDatablocks); + + TCase* tc_LinkedDatabase = tcase_create("LinkedDatabase"); + tcase_add_test(tc_LinkedDatabase, test_LinkedDatabase); + + TCase* tc_ReadOnlyDatabase = tcase_create("ReadOnlyDatabase"); + tcase_add_test(tc_ReadOnlyDatabase, test_ReadOnlyDatabase); + + TCase* tc_MultipleWrites = tcase_create("MultipleWrites"); + tcase_add_test(tc_MultipleWrites, test_MultipleWrites); + + TCase* tc_WriteThrough = tcase_create("WriteThrough"); + tcase_add_test(tc_WriteThrough, test_WriteThrough); + tcase_set_timeout(tc_WriteThrough, 20); + + + + suite_add_tcase(s, tc_persOpenLocalDB); + tcase_add_checked_fixture(tc_persOpenLocalDB, data_setup, data_teardown); + + suite_add_tcase(s, tc_persOpenRCT); + tcase_add_checked_fixture(tc_persOpenRCT, data_setup, data_teardown); + + suite_add_tcase(s, tc_persSetDataLocalDB); + tcase_add_checked_fixture(tc_persSetDataLocalDB, data_setup, data_teardown); + + suite_add_tcase(s, tc_persGetDataLocalDB); + tcase_add_checked_fixture(tc_persGetDataLocalDB, data_setup, data_teardown); + + suite_add_tcase(s, tc_persSetDataRCT); + tcase_add_checked_fixture(tc_persSetDataRCT, data_setup, data_teardown); + + suite_add_tcase(s, tc_persGetDataRCT); + tcase_add_checked_fixture(tc_persGetDataRCT, data_setup, data_teardown); + + suite_add_tcase(s, tc_persGetDataSize); + tcase_add_checked_fixture(tc_persGetDataSize, data_setup, data_teardown); + + suite_add_tcase(s, tc_persDeleteDataLocalDB); + tcase_add_checked_fixture(tc_persDeleteDataLocalDB, data_setup, data_teardown); + + suite_add_tcase(s, tc_persDeleteDataRct); + tcase_add_checked_fixture(tc_persDeleteDataRct, data_setup, data_teardown); + + suite_add_tcase(s, tc_persGetKeyListSizeLocalDB); + tcase_add_checked_fixture(tc_persGetKeyListSizeLocalDB, data_setup, data_teardown); + + suite_add_tcase(s, tc_persGetKeyListLocalDB); + tcase_add_checked_fixture(tc_persGetKeyListLocalDB, data_setup, data_teardown); + + suite_add_tcase(s, tc_persGetResourceListSizeRct); + tcase_add_checked_fixture(tc_persGetResourceListSizeRct, data_setup, data_teardown); + + suite_add_tcase(s, tc_persGetResourceListRct); + tcase_add_checked_fixture(tc_persGetResourceListRct, data_setup, data_teardown); + + suite_add_tcase(s, tc_persCacheSize); //do not run when using writethrough + tcase_add_checked_fixture(tc_persCacheSize, data_setup, data_teardown); + + suite_add_tcase(s, tc_persCachedConcurrentAccess); + tcase_add_checked_fixture(tc_persCachedConcurrentAccess, data_setup, data_teardown); + //suite_add_tcase(s, tc_persCachedConcurrentAccess2); + + suite_add_tcase(s, tc_BadParameters); + tcase_add_checked_fixture(tc_BadParameters, data_setup, data_teardown); + + suite_add_tcase(s, tc_RebuildHashtables); + tcase_add_checked_fixture(tc_RebuildHashtables, data_setup, data_teardown); + + suite_add_tcase(s, tc_RecoverDatablocks); //add test for writethrough (writeback order is different when using cache) + tcase_add_checked_fixture(tc_RecoverDatablocks, data_setup, data_teardown); + + suite_add_tcase(s, tc_LinkedDatabase); + tcase_add_checked_fixture(tc_LinkedDatabase, data_setup, data_teardown); + + suite_add_tcase(s, tc_ReadOnlyDatabase); + tcase_add_checked_fixture(tc_ReadOnlyDatabase, data_setup, data_teardown); + + suite_add_tcase(s, tc_MultipleWrites); + tcase_add_checked_fixture(tc_MultipleWrites, data_setup, data_teardown); + + suite_add_tcase(s, tc_WriteThrough); + tcase_add_checked_fixture(tc_WriteThrough, data_setup, data_teardown); + + return s; +} + + +int main(int argc, char* argv[]) +{ + int nr_failed = 0, nr_run = 0, i = 0; + TestResult** tResult; + +#if 1 + Suite* s = persistenceCommonLib_suite(); + SRunner* sr = srunner_create(s); + srunner_set_xml(sr, "/tmp/persistenceCommonObjectTest.xml"); + srunner_set_log(sr, "/tmp/persistenceCommonObjectTest.log"); + srunner_run_all(sr, /*CK_NORMAL*/CK_VERBOSE); + + nr_failed = srunner_ntests_failed(sr); + nr_run = srunner_ntests_run(sr); + + tResult = srunner_results(sr); + for (i = 0; i < nr_run; i++) + { + (void) tr_rtype(tResult[i]); // get status of each test + } + + srunner_free(sr); +#endif + + dlt_free(); + return (0 == nr_failed) ? EXIT_SUCCESS : EXIT_FAILURE; +} + -- cgit v1.2.1