diff options
| author | Jay Doane <jaydoane@apache.org> | 2020-04-15 10:19:58 -0700 |
|---|---|---|
| committer | Jay Doane <jay.s.doane@gmail.com> | 2020-04-15 10:21:55 -0700 |
| commit | 36364516d32a368b5c58ee197f9c3fbb82394f81 (patch) | |
| tree | a29bbb8d5e1e03d7b3eb95c58687deca54ee4450 | |
| parent | 6bc6f9c285acba015614e08a62edffbd9a2f3699 (diff) | |
| download | couchdb-36364516d32a368b5c58ee197f9c3fbb82394f81.tar.gz | |
Enable configurable binary chunk size
Currently, the size of binary chunks used for values is fixed at the FDB
imposed limit of 100kB, although they recommend using 10KB [1], (also
note they subtly change units).
This makes that value configurable, allowing e.g. benchmarks to compare
performance of runs with varying chunk size. The cost is a ~10µs config
lookup penalty each time data needs to be chunked.
[1] https://www.foundationdb.org/files/record-layer-paper.pdf
| -rw-r--r-- | rel/overlay/etc/default.ini | 3 | ||||
| -rw-r--r-- | src/fabric/include/fabric2.hrl | 2 | ||||
| -rw-r--r-- | src/fabric/src/fabric2_fdb.erl | 15 |
3 files changed, 16 insertions, 4 deletions
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index e10a5a0c7..dfc67f7fb 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -239,6 +239,9 @@ port = 6984 ; ; Enable or disable automatic stale index removal in the auto-updater ;index_updater_remove_old_indices = false +; +; Byte size of binary chunks written to FDB values. Defaults to FDB max limit. +;binary_chunk_size = 100000 ; [rexi] ; buffer_count = 2000 diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl index 587b4f888..2e588f8a3 100644 --- a/src/fabric/include/fabric2.hrl +++ b/src/fabric/include/fabric2.hrl @@ -77,4 +77,4 @@ -define(TRANSACTION_CANCELLED, 1025). --define(BINARY_CHUNK_SIZE, 100000). +-define(DEFAULT_BINARY_CHUNK_SIZE, 100000). diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl index d96c3ae60..53102d6e9 100644 --- a/src/fabric/src/fabric2_fdb.erl +++ b/src/fabric/src/fabric2_fdb.erl @@ -1628,12 +1628,16 @@ sum_rem_rev_sizes(RevInfos) -> chunkify_binary(Data) -> + chunkify_data(Data, binary_chunk_size()). + + +chunkify_data(Data, Size) -> case Data of <<>> -> []; - <<Head:?BINARY_CHUNK_SIZE/binary, Rest/binary>> -> - [Head | chunkify_binary(Rest)]; - <<_/binary>> when size(Data) < ?BINARY_CHUNK_SIZE -> + <<Head:Size/binary, Rest/binary>> -> + [Head | chunkify_data(Rest, Size)]; + <<_/binary>> when size(Data) < Size -> [Data] end. @@ -1988,6 +1992,11 @@ get_info_wait_int(#info_future{} = InfoFuture) -> [CProp | MProps]. +binary_chunk_size() -> + config:get_integer( + "fabric", "binary_chunk_size", ?DEFAULT_BINARY_CHUNK_SIZE). + + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). |