Tarball conversion

1 files changed, 556 insertions, 0 deletions

diff --git a/examples/java/src/db/BulkExample.java b/examples/java/src/db/BulkExample.java
new file mode 100644
index 00000000..1d752ff4
--- /dev/null
+++ b/examples/java/src/db/BulkExample.java
@@ -0,0 +1,556 @@
+/*-
+ * See the file LICENSE for redistribution information.
+ *
+ * Copyright (c) 1997, 2012 Oracle and/or its affiliates.  All rights reserved.
+ *
+ * $Id$
+ */
+
+package db;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import com.sleepycat.db.*;
+
+public class BulkExample {
+    private static final String dbFileName = "BulkExample.db";
+    private static final String pDbName = "primary";
+    private static final String sDbName = "secondary";
+    private static final String home = "BulkExample";
+    private static final int buffLen = 1024 * 1024;
+
+    /* Bulk methods demonstrated in the example */
+    private enum Operations {BULK_DELETE, BULK_READ, BULK_UPDATE}
+
+    private Database pdb;
+    private Environment env;
+    private Operations opt = Operations.BULK_READ;
+    private SecondaryDatabase sdb;
+    private int cachesize = 0, dups = 0, pagesize = 0;
+    private int num = 1000000;
+    private boolean secondary = false;
+
+    public static void main(String[] args) {
+        long startTime, endTime;
+        int count;
+
+        BulkExample app = new BulkExample();
+
+        /* Parse argument */
+        for (int i = 0; i < args.length; i++) {
+            if (args[i].compareTo("-c") == 0) {
+                if (i == args.length - 1)
+                    usage();
+                i++;
+                app.cachesize = Integer.parseInt(args[i]);
+                System.out.println("[CONFIG] cachesize " + app.cachesize);
+            } else if (args[i].compareTo("-d") == 0) {
+                if (i == args.length - 1)
+                    usage();
+                i++;
+                app.dups = Integer.parseInt(args[i]);
+                System.out.println(
+                    "[CONFIG] number of duplicates " + app.dups);
+            } else if (args[i].compareTo("-n") == 0) {
+                if (i == args.length - 1)
+                    usage();
+                i++;
+                app.num = Integer.parseInt(args[i]);
+                System.out.println("[CONFIG] number of keys " + app.num);
+            } else if (args[i].compareTo("-p") == 0) {
+                if (i == args.length - 1)
+                    usage();
+                i++;
+                app.pagesize = Integer.parseInt(args[i]);
+                System.out.println("[CONFIG] pagesize " + app.pagesize);
+            } else if (args[i].compareTo("-D") == 0) {
+                app.opt = Operations.BULK_DELETE;
+            } else if (args[i].compareTo("-R") == 0) {
+                app.opt = Operations.BULK_READ;
+            } else if (args[i].compareTo("-S") == 0) {
+                app.secondary = true;
+            } else if (args[i].compareTo("-U") == 0) {
+                app.opt = Operations.BULK_UPDATE;
+            }else
+                usage();
+        }
+
+        /* Open environment and database(s) */
+        try {
+            /* If perform bulk update or delete, clean environment home */
+            app.cleanHome(app.opt != Operations.BULK_READ);
+            app.initDbs();
+        }catch (Exception e) {
+            e.printStackTrace();
+            app.closeDbs();
+        }
+
+        try {
+            /* Perform bulk read from existing primary db */
+            if (app.opt == Operations.BULK_READ) {
+                startTime = System.currentTimeMillis();
+                count = app.bulkRead();
+                endTime = System.currentTimeMillis();
+                app.printBulkOptStat(
+                    Operations.BULK_READ, count, startTime, endTime);
+            } else {
+                /* Perform bulk update to populate the db */
+                startTime = System.currentTimeMillis();
+                app.bulkUpdate();
+                endTime = System.currentTimeMillis();
+                count = (app.dups == 0) ? app.num : app.num * app.dups;
+                app.printBulkOptStat(
+                    Operations.BULK_UPDATE, count, startTime, endTime);
+
+                /* Perform bulk delete in primary db or secondary db */
+                if (app.opt == Operations.BULK_DELETE) {
+                    startTime = System.currentTimeMillis();
+                    if (app.secondary == false) {
+                        /*
+                         * Delete from the first key to a random one in
+                         * primary db.
+                         */
+                        count = new java.util.Random().nextInt(app.num);
+                        count = app.bulkDelete(count);
+                    } else {
+                        /*
+                         * Delete from the first key to a random one in
+                         * secondary db.
+                         */
+                        count = new java.util.Random().nextInt(
+                            DataVal.tstring.length());
+                        count = app.bulkSecondaryDelete(
+                            (DataVal.tstring.getBytes())[count], 
+                            true);
+                    } 
+                    endTime = System.currentTimeMillis();
+
+                    app.printBulkOptStat(Operations.BULK_DELETE, 
+                        count, startTime, endTime);
+                }
+            }
+        } catch (DatabaseException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            /* Close dbs */
+            app.closeDbs();
+        }
+    }
+
+    /* Perform bulk delete in primary db */
+    public int bulkDelete(int value) throws DatabaseException, IOException {
+        DatabaseEntry key = new DatabaseEntry();
+        Transaction txn = env.beginTransaction(null, null);
+        int count = 0;
+
+        try {
+            if (dups != 0) {
+                /* Delete a set of keys */
+                MultipleDataEntry keySet = new MultipleDataEntry();
+                keySet.setData(new byte[buffLen]);
+                keySet.setUserBuffer(buffLen, true);
+                for (int i = 0; i < value; i++) {
+                    key.setData(intToByte(i));
+                    keySet.append(key);
+                }
+                if (pdb.deleteMultiple(txn, keySet) != 
+                    OperationStatus.SUCCESS)
+                    count = 0;
+                else
+                    count = value * dups;
+            } else {
+                /* Delete a set of key/value pairs */
+                MultipleKeyDataEntry pairSet = new MultipleKeyDataEntry();
+                pairSet.setData(new byte[buffLen * 2]);
+                pairSet.setUserBuffer(buffLen * 2, true);
+                DatabaseEntry data = new DatabaseEntry(); 
+                for (int i = 0; i < value; i++) {
+                    key.setData(intToByte(i));
+                    DataVal dataVal = new DataVal(i);
+                    data.setData(dataVal.getBytes());
+                    pairSet.append(key, data);
+                }
+                if (pdb.deleteMultipleKey(txn, pairSet) != 
+                    OperationStatus.SUCCESS)
+                    count = 0;
+                else
+                    count = value;
+            }
+
+            txn.commit();
+            return count;
+        } catch (DatabaseException e) {
+            txn.abort();
+            throw e;
+        }
+    }
+
+    /* Perform bulk read in primary db */
+    public int bulkRead() throws DatabaseException {
+        ByteBuffer buffer1, buffer2;
+        Cursor cursor;
+        DatabaseEntry key, data;
+        MultipleKeyNIODataEntry pairs;
+        MultipleNIODataEntry keySet, dataSet;
+        int count;
+
+        key = new DatabaseEntry();
+        data = new DatabaseEntry();
+        count = 0;
+        
+        Transaction txn = env.beginTransaction(null, null);
+
+        try {
+            cursor = pdb.openCursor(txn, null);
+        } catch (DatabaseException e) {
+            txn.abort();
+            return 0;
+        }
+
+        try {    
+            if (dups == 0){
+                /* Get all records in a single key/data buffer */
+                pairs = new MultipleKeyNIODataEntry();
+                buffer1 = ByteBuffer.allocateDirect(buffLen * 2);
+                pairs.setDataNIO(buffer1);
+                pairs.setUserBuffer(buffLen * 2, true);
+                while (cursor.getNext(key, pairs, null) ==
+                    OperationStatus.SUCCESS) {
+                    while(pairs.next(key, data))
+                        count++;
+                    key = new DatabaseEntry();
+                }
+            } else {
+                /*
+                 * Get all key/data pairs in two buffers, one for all keys,
+                 * the other for all data.
+                 */
+                keySet = new MultipleNIODataEntry();
+                buffer1 = ByteBuffer.allocateDirect(buffLen);
+                keySet.setDataNIO(buffer1);
+                keySet.setUserBuffer(buffLen, true);
+                dataSet = new MultipleNIODataEntry();
+                buffer2 = ByteBuffer.allocateDirect(buffLen);
+                dataSet.setDataNIO(buffer2);
+                dataSet.setUserBuffer(buffLen, true);
+                while (cursor.getNext(keySet, dataSet, null) == 
+                    OperationStatus.SUCCESS)
+                    while(keySet.next(key) && dataSet.next(data))
+                        count++;
+            }
+            cursor.close();
+            txn.commit();
+            return count;
+        } catch (DatabaseException e) {
+            cursor.close();
+            txn.abort();
+            throw e;
+        }
+    }
+
+    /* Perform bulk delete in secondary db */
+    public int bulkSecondaryDelete(byte value, boolean deletePair)
+        throws DatabaseException, IOException {
+        DatabaseEntry key = new DatabaseEntry();
+        byte[] valueBytes = new byte[1];
+        byte[] tstrBytes = DataVal.tstring.getBytes();
+        int i = 0;
+        int count = 0;
+
+        Transaction txn = env.beginTransaction(null, null);
+
+        try {
+            /* Prepare the buffer for a set of keys */
+            MultipleDataEntry keySet = new MultipleDataEntry();
+            keySet.setData(new byte[buffLen]);
+            keySet.setUserBuffer(buffLen, true);
+
+            /* Delete the given key and all keys prior to it */
+            do {
+                valueBytes[0] = tstrBytes[i];
+                key.setData(valueBytes);
+                keySet.append(key);
+            } while (value != tstrBytes[i++]);
+            if (sdb.deleteMultiple(txn, keySet) != OperationStatus.SUCCESS)
+                count = 0;
+            else
+                count = this.num / DataVal.tstring.length() * i;
+            if (i < this.num % DataVal.tstring.length())
+                count += i;
+
+            txn.commit();
+            return count;
+        } catch (DatabaseException e) {
+            txn.abort();
+            throw e;
+        }
+    }
+
+    /* Perform bulk update in primary db */
+    public void bulkUpdate() throws DatabaseException, IOException {
+        DatabaseEntry key, data;
+        MultipleDataEntry keySet, dataSet;
+        MultipleKeyDataEntry pairSet;
+        DataVal dataVal;
+        int i, j;
+
+        Transaction txn = env.beginTransaction(null, null);
+        key = new DatabaseEntry();
+        data = new DatabaseEntry();
+
+        try {
+            if (this.dups == 0) {
+                /* 
+                 * Bulk insert non-duplicate key/data pairs. Those pairs
+                 * are filled into one single buffer.
+                 */ 
+                pairSet = new MultipleKeyDataEntry();
+                pairSet.setData(new byte[buffLen * 2]);
+                pairSet.setUserBuffer(buffLen * 2, true);
+                for (i = 0; i < this.num; i++) {
+                    dataVal = new DataVal(i);
+                    key.setData(intToByte(i));
+                    data.setData(dataVal.getBytes());
+                    pairSet.append(key, data);
+                }
+                pdb.putMultipleKey(txn, pairSet, true);
+            } else {
+                /* Bulk insert duplicate key/data pairs. All keys are  
+                 * filled into a buffer, and their data is filled into 
+                 * another buffer.
+                 */ 
+                keySet = new MultipleDataEntry();
+                keySet.setData(new byte[buffLen]);
+                keySet.setUserBuffer(buffLen, true);
+                dataSet = new MultipleDataEntry();
+                dataSet.setData(new byte[buffLen]);
+                dataSet.setUserBuffer(buffLen, true);
+
+                for (i = 0; i < this.num; i++) {
+                    j = 0;
+                    dataVal = new DataVal(i);
+                    do {
+                        key.setData(intToByte(i));
+                        keySet.append(key);
+                        dataVal.setDataId(j);
+                        data.setData(dataVal.getBytes());
+                        dataSet.append(data);
+                    } while(++j < this.dups);
+                }
+                pdb.putMultiple(txn, keySet, dataSet, true);
+            }
+
+            txn.commit();
+        }catch (DatabaseException e1) {
+            txn.abort();
+            throw e1;
+        }catch (IOException e2) {
+            txn.abort();
+            throw e2;
+        }
+    }
+
+    /* Clean the home directory */
+    public void cleanHome(boolean clean) {
+        File file = new File(home);
+        /* 
+         * If the home directory doesn't exist, create the directory. 
+         * Unless, if it is requested to be cleaned, delete all and 
+         * create a new one.
+         */ 
+        if (!file.exists())
+            file.mkdir();
+        else if (clean == true) {
+            File[] files = file.listFiles();
+            for(int i=0; i< files.length; i++) {
+                files[i].delete();
+            }
+            file.delete();
+            file.mkdir();
+        }
+    }
+
+    /* Close database(s) and environment */
+    public void closeDbs() {
+        try {
+            if (this.secondary)
+                sdb.close();
+            pdb.close();
+            env.close();
+        } catch (DatabaseException e) {
+            System.exit(1);
+        }
+    }
+
+    /* Print statistics in bulk operations */
+    public void printBulkOptStat(
+        Operations opt, int count, long startTime, long endTime) {
+        System.out.print("[STAT] "); 
+        if (opt == Operations.BULK_DELETE && !secondary)
+            System.out.print("Bulk delete ");
+        else if (opt == Operations.BULK_DELETE && secondary)
+            System.out.print("Bulk secondary delete ");
+        else if (opt == Operations.BULK_READ)
+            System.out.print("Bulk read ");
+        else
+            System.out.print("Bulk update ");
+            
+        System.out.print(count + " records");
+        System.out.print(" in " + (endTime - startTime) + " ms");
+        System.out.println(", that is " + 
+            (int)((double)1000 / (endTime - startTime) * (double)count) +
+            " records/second");
+    }
+
+    /* Initialize environment and database (s) */
+    public void initDbs() throws DatabaseException, FileNotFoundException {
+        /* Open transactional environment */
+        EnvironmentConfig envConfig = new EnvironmentConfig();
+        envConfig.setAllowCreate(true);
+        envConfig.setInitializeCache(true);
+        envConfig.setInitializeLocking(true);
+        envConfig.setInitializeLogging(true);
+        envConfig.setTransactional(true);
+        File envFile = new File(home);
+        try {
+            env = new Environment(envFile, envConfig);
+        } catch (Exception e) {
+            System.exit(1);
+        }
+
+        Transaction txn = env.beginTransaction(null, null);
+
+        try {
+            /* Open primary db in transaction */
+            DatabaseConfig dbConfig = new DatabaseConfig();
+            dbConfig.setTransactional(true);
+            dbConfig.setAllowCreate(true);
+            dbConfig.setCacheSize(this.cachesize);
+            dbConfig.setPageSize(this.pagesize);
+            dbConfig.setType(DatabaseType.BTREE);
+            if (this.dups != 0)
+                dbConfig.setUnsortedDuplicates(true);
+            pdb = env.openDatabase(txn, dbFileName, pDbName, dbConfig);
+
+            /* Open secondary db in transaction */
+            if (this.secondary) {
+                SecondaryConfig sdbConfig = new SecondaryConfig();
+                sdbConfig.setTransactional(true);
+                sdbConfig.setAllowCreate(true);
+                sdbConfig.setCacheSize(this.cachesize);
+                sdbConfig.setPageSize(this.pagesize);
+                sdbConfig.setType(DatabaseType.BTREE);
+                sdbConfig.setKeyCreator(new FirstStrKeyCreator());
+                sdbConfig.setSortedDuplicates(true);
+                sdb = env.openSecondaryDatabase(
+                    txn, dbFileName, sDbName, pdb, sdbConfig);
+            }
+
+            txn.commit();
+        } catch (DatabaseException e1) {
+            txn.abort();
+            throw e1;
+        } catch (FileNotFoundException e2) {
+            txn.abort();
+            throw e2;
+        }
+    }
+
+    /* Convert integer to byte array */
+    public byte[] intToByte(int i) {
+        byte[] dword = new byte[4];
+        dword[0] = (byte) (i & 0x00FF);
+        dword[1] = (byte) ((i >> 8) & 0x000000FF);
+        dword[2] = (byte) ((i >> 16) & 0x000000FF);
+        dword[3] = (byte) ((i >> 24) & 0x000000FF);
+        return dword;
+    }
+
+    /* Usage of BulkExample */
+    private static void usage() {
+        System.out.println("Usage: BulkExample \n" +
+            " -c cachesize [1000 * pagesize] \n" + 
+            " -d number of duplicates [none] \n" + 
+            " -n number of keys [1000000] \n" + 
+            " -p pagesize [65536] \n" +
+            " -D perform bulk delete \n" +
+            " -R perform bulk read \n" + 
+            " -S perform bulk read in secondary database \n" + 
+            " -U perform bulk update \n");
+        System.exit(1);
+    }
+
+    class DataVal {
+        /* String template */
+        public static final String tstring = "0123456789abcde";
+
+        private String dataString;
+        private int dataId;
+
+        /* 
+         * Construct DataVal with given kid and id. The dataString is the
+         * rotated tstring from the kid position. The dataId is the id.
+         */
+        public DataVal(int kid) {
+            int rp = kid % tstring.length();
+            if (rp == 0)
+                this.dataString = tstring;
+            else
+                this.dataString = tstring.substring(rp) +
+                    tstring.substring(0, rp - 1);
+
+            this.dataId = 0;
+        }
+
+        /* Parse the object including dataId and dataString to byte array. */
+        public byte[] getBytes() throws IOException {
+            int i;
+
+            byte[] strBytes = dataString.getBytes();
+            byte[] intBytes = intToByte(dataId);
+            byte[] bytes = new byte[4 + strBytes.length];
+            for (i = 0; i < 4 + strBytes.length; i++) {
+                if (i < 4)
+                    bytes[i] = intBytes[i];
+                else
+                    bytes[i] = strBytes[i - 4];
+            }
+
+            return bytes;
+        }
+
+        public int getDataId() {
+            return dataId;
+        }
+            
+        public void setDataId(int id) {
+            dataId = id;
+        }
+
+        public String getDataString() {
+            return dataString;
+        }
+    }
+
+    /* Secondary key creator */
+    class FirstStrKeyCreator implements SecondaryKeyCreator {
+        public boolean createSecondaryKey(SecondaryDatabase secondary,
+                DatabaseEntry key, DatabaseEntry data, DatabaseEntry result)
+                throws DatabaseException {
+            /*
+             * Create the secondary key. It would be the first character in
+             * dataString of DataVal, that is the 5th byte in data. 
+             */
+            byte[] firstStr = new byte[1];
+            firstStr[0] = (data.getData())[4];
+            result.setData(firstStr);
+            result.setSize(1);
+            return true;
+        }
+    }
+}