diff --git a/docs/PATHS.md b/docs/PATHS.md index a1f30bf5..f9f4c5ae 100644 --- a/docs/PATHS.md +++ b/docs/PATHS.md @@ -34,7 +34,7 @@ If the lookup does not return an entry, then the Bookie must request that the Pe The Penciller on receiving a request to lookup a Key (with hash), needs to check first in its in-memory view of level-zero. Level zero is formed of a: -- a list of trees, with each tree being the output of an export of a Ledger Cache (it is exported using ets:tab2list, and then converted to a tree using leveled_tree:from_orderedset). +- a list of trees, with each tree being the output of an export of a Ledger Cache (it is exported using ets:tab2list, and then converted to a tree using leveled_tree:from_ets). - an index of hashes, which is an array that has accumulated the index arrays formed within the Bookie's Ledger cache. The index of hashes is used to find which of the trees in the list of trees that the Key may be present in. The hash is converted into a slot and lookup hash part, and the binary is pulled from the entry in the array that aligns with the slot. diff --git a/include/leveled.hrl b/include/leveled.hrl index 4818e61a..ec77b7cd 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -11,7 +11,8 @@ %%% Configurable startup defaults %%%============================================================================ -define(CACHE_SIZE, 2500). --define(MAX_CACHE_MULTTIPLE, 2). +-define(MAX_CACHE_MULTIPLE, 2). +-define(MAX_SQN_MULTIPLE, 4). -define(MIN_CACHE_SIZE, 100). -define(MIN_PCL_CACHE_SIZE, 400). -define(MAX_PCL_CACHE_SIZE, 28000). @@ -32,6 +33,7 @@ -define(DEFAULT_STATS_PERC, 10). -define(DEFAULT_SYNC_STRATEGY, none). -define(DEFAULT_BLOCK_VERSION, 1). +-define(LEDGER_VALUE_VERSION, 3). %%%============================================================================ %%%============================================================================ @@ -49,7 +51,7 @@ -define(MIN_KEYCHECK_FREQUENCY, 1). -define(MAX_LEVELS, 8). %% Should equal the length of the LEVEL_SCALEFACTOR --define(CACHE_TYPE, skpl). +-define(CACHE_TYPE, idxt). %%%============================================================================ %%%============================================================================ diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index 4e6dae7b..92b3a20c 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -115,7 +115,7 @@ {root_path, undefined}, {snapshot_bookie, undefined}, {cache_size, ?CACHE_SIZE}, - {cache_multiple, ?MAX_CACHE_MULTTIPLE}, + {cache_multiple, ?MAX_CACHE_MULTIPLE}, {max_journalsize, 1000000000}, {max_journalobjectcount, 200000}, {max_sstslots, 256}, @@ -132,6 +132,7 @@ {ledger_preloadpagecache_level, ?SST_PAGECACHELEVEL_LOOKUP}, {compression_method, ?COMPRESSION_METHOD}, {ledger_compression, as_store}, + {ledger_value_version, ?LEDGER_VALUE_VERSION}, {block_version, 1}, {compression_point, ?COMPRESSION_POINT}, {compression_level, ?COMPRESSION_LEVEL}, @@ -152,8 +153,8 @@ tuple() | empty_cache, load_queue = [] :: list(), index = leveled_pmem:new_index(), - min_sqn = infinity :: integer() | infinity, - max_sqn = 0 :: integer() + min_sqn = infinity :: non_neg_integer() | infinity, + max_sqn = 0 :: non_neg_integer() }). -record(state, { @@ -167,6 +168,8 @@ head_only = false :: boolean(), head_lookup = true :: boolean(), ink_checking = ?MAX_KEYCHECK_FREQUENCY :: integer(), + ledger_value_version = ?LEDGER_VALUE_VERSION :: + leveled_codec:ledger_value_version(), bookie_monref :: reference() | undefined, monitor = {no_monitor, 0} :: leveled_monitor:monitor() }). @@ -218,17 +221,17 @@ % should be partial) | {sync_strategy, sync_mode()} % Should be sync if it is necessary to flush to disk after every - % write, or none if not (allow the OS to schecdule). This has a + % write, or none if not (allow the OS to schedule). This has a % significant impact on performance which can be mitigated % partially in hardware (e.g through use of FBWC). - % riak_sync is used for backwards compatability with OTP16 - and + % riak_sync is used for backwards compatibility with OTP16 - and % will manually call sync() after each write (rather than use the % O_SYNC option on startup) | {head_only, false | with_lookup | no_lookup} % When set to true, there are three fundamental changes as to how % leveled will work: - % - Compaction of the journalwill be managed by simply removing any - % journal file thathas a highest sequence number persisted to the + % - Compaction of the journal will be managed by simply removing any + % journal file that has a highest sequence number persisted to the % ledger; % - GETs are not supported, only head requests; % - PUTs should arrive batched object specs using the book_mput/2 @@ -317,6 +320,9 @@ % Define an alternative to the compression method to be used by the % ledger only. Default is as_store - use the method defined as % compression_method for the whole store + | {ledger_value_version, 2 | 3} + % version 2 uses a tuple for the value in the ledger, whereas version + % 3 has a purely binary value to support direct decoding | {block_version, 0 | 1} % Version of the leveled_sst blocks. Block version 0 does not use % sub-blocks, whereas block version 1 has multiple types of blocks @@ -387,9 +393,9 @@ integer() }. +%% erlfmt:ignore format avoids issues with VSCode/ELP -type initial_loadfun() :: - fun( - ( + fun(( leveled_codec:journal_key(), dynamic(), non_neg_integer(), @@ -1246,7 +1252,7 @@ book_islastcompactionpending(Pid) -> %% @doc Trim the journal when in head_only mode %% -%% In head_only mode the journlacna be trimmed of entries which are before the +%% In head_only mode the journal can be trimmed of entries which are before the %% persisted SQN. This is much quicker than compacting the journal book_trimjournal(Pid) -> @@ -1360,6 +1366,8 @@ init([Opts]) -> leveled_log:add_forcedlogs(ForcedLogs), DatabaseID = proplists:get_value(database_id, Opts), leveled_log:set_databaseid(DatabaseID), + LedgerValueVersion = + proplists:get_value(ledger_value_version, Opts), {ok, Monitor} = leveled_monitor:monitor_start( @@ -1422,7 +1430,8 @@ init([Opts]) -> PencillerOpts0 = PencillerOpts#penciller_options{sst_options = SSTOpts0}, - {Inker, Penciller} = startup(InkerOpts, PencillerOpts0), + {Inker, Penciller} = + startup(InkerOpts, PencillerOpts0, LedgerValueVersion), NewETS = ets:new(mem, [ordered_set]), ?STD_LOG(b0001, [Inker, Penciller]), @@ -1434,6 +1443,7 @@ init([Opts]) -> head_lookup = HeadLookup, inker = Inker, penciller = Penciller, + ledger_value_version = LedgerValueVersion, ledger_cache = #ledger_cache{mem = NewETS}, monitor = {Monitor, StatLogFrequency} }}; @@ -1476,7 +1486,13 @@ handle_call( {T0, SW1} = leveled_monitor:step_time(SW0), Changes = preparefor_ledgercache( - null, LedgerKey, SQN, Object, ObjSize, {IndexSpecs, TTL} + null, + LedgerKey, + SQN, + Object, + ObjSize, + {IndexSpecs, TTL}, + State#state.ledger_value_version ), {T1, SW2} = leveled_monitor:step_time(SW1), Cache0 = addto_ledgercache(Changes, State#state.ledger_cache), @@ -1515,7 +1531,8 @@ handle_call({mput, ObjectSpecs, TTL}, From, State) when SQN, null, length(ObjectSpecs), - {ObjectSpecs, TTL} + {ObjectSpecs, TTL}, + State#state.ledger_value_version ), Cache0 = addto_ledgercache(Changes, State#state.ledger_cache), case State#state.slow_offer of @@ -1554,8 +1571,8 @@ handle_call({get, Bucket, Key, Tag}, _From, State) when not_present -> not_found; Head -> - {Seqn, Status, _MH, _MD} = - leveled_codec:striphead_to_v1details(Head), + {Status, Seqn} = + leveled_codec:ledgermd_statussqn(Head), case Status of tomb -> not_found; @@ -1612,10 +1629,10 @@ handle_call({head, Bucket, Key, Tag, SQNOnly}, _From, State) when not_present -> {not_found, null, JrnalCheckFreq}; Head -> - case leveled_codec:striphead_to_v1details(Head) of - {_SeqN, tomb, _MH, _MD} -> + case leveled_codec:ledgermd_statussqnumd(Head) of + {tomb, _Seqn, _MD} -> {not_found, null, JrnalCheckFreq}; - {SeqN, {active, TS}, _MH, MD} -> + {{active, TS}, SeqN, MD} -> case TS >= leveled_util:integer_now() of true -> I = State#state.inker, @@ -1843,13 +1860,14 @@ empty_ledgercache() -> pid(), list(load_item()), ledger_cache(), - leveled_codec:compaction_strategy() + leveled_codec:compaction_strategy(), + leveled_codec:ledger_value_version() ) -> ledger_cache(). %% @doc %% The push to penciller must start as a tree to correctly de-duplicate %% the list by order before becoming a de-duplicated list for loading -push_to_penciller(Penciller, LoadItemList, LedgerCache, ReloadStrategy) -> +push_to_penciller(Penciller, LoadItemList, LedgerCache, ReloadStrategy, VV) -> UpdLedgerCache = lists:foldl( fun({InkTag, PK, SQN, Obj, IndexSpecs, ValSize}, AccLC) -> @@ -1864,11 +1882,18 @@ push_to_penciller(Penciller, LoadItemList, LedgerCache, ReloadStrategy) -> ValSize, IndexSpecs, AccLC, - Penciller + Penciller, + VV ); _ -> preparefor_ledgercache( - InkTag, PK, SQN, Obj, ValSize, IndexSpecs + InkTag, + PK, + SQN, + Obj, + ValSize, + IndexSpecs, + VV ) end, addto_ledgercache(Chngs, AccLC, loader) @@ -2004,13 +2029,18 @@ fetch_value(Inker, {Key, SQN}) -> %%% Internal functions %%%============================================================================ --spec startup(#inker_options{}, #penciller_options{}) -> {pid(), pid()}. +-spec startup( + #inker_options{}, + #penciller_options{}, + leveled_codec:ledger_value_version() +) -> + {pid(), pid()}. %% @doc %% Startup the Inker and the Penciller, and prompt the loading of the Penciller %% from the Inker. The Penciller may be shutdown without the latest data %% having been persisted: and so the Iker must be able to update the Penciller %% on startup with anything that happened but wasn't flushed to disk. -startup(InkerOpts, PencillerOpts) -> +startup(InkerOpts, PencillerOpts, VV) -> {ok, Inker} = leveled_inker:ink_start(InkerOpts), {ok, Penciller} = leveled_penciller:pcl_start(PencillerOpts), LedgerSQN = leveled_penciller:pcl_getstartupsequencenumber(Penciller), @@ -2020,7 +2050,7 @@ startup(InkerOpts, PencillerOpts) -> BatchFun = fun(BatchAcc, Acc) -> push_to_penciller( - Penciller, BatchAcc, Acc, ReloadStrategy + Penciller, BatchAcc, Acc, ReloadStrategy, VV ) end, InitAccFun = @@ -2588,10 +2618,7 @@ readycache_forsnapshot(LedgerCache, {StartKey, EndKey}) -> end; readycache_forsnapshot(LedgerCache, Query) -> % Need to convert the Ledger Cache away from using the ETS table - Tree = leveled_tree:from_orderedset( - LedgerCache#ledger_cache.mem, - ?CACHE_TYPE - ), + Tree = leveled_tree:from_ets(LedgerCache#ledger_cache.mem, ?CACHE_TYPE), case leveled_tree:tsize(Tree) of 0 -> #ledger_cache{ @@ -2636,7 +2663,7 @@ scan_table(Table, StartKey, EndKey) -> [] -> scan_table(Table, StartKey, EndKey, [], infinity, 0); [{StartKey, StartVal}] -> - SQN = leveled_codec:strip_to_seqonly({StartKey, StartVal}), + SQN = leveled_codec:ledgermd_sqn(StartVal), scan_table( Table, StartKey, @@ -2657,7 +2684,7 @@ scan_table(Table, StartKey, EndKey, Acc, MinSQN, MaxSQN) -> {lists:reverse(Acc), MinSQN, MaxSQN}; false -> [{NextKey, NextVal}] = ets:lookup(Table, NextKey), - SQN = leveled_codec:strip_to_seqonly({NextKey, NextVal}), + SQN = leveled_codec:ledgermd_sqn(NextVal), scan_table( Table, NextKey, @@ -2738,7 +2765,8 @@ check_notfound(CheckFrequency, CheckFun) -> non_neg_integer(), any(), integer(), - leveled_codec:journal_keychanges() + leveled_codec:journal_keychanges(), + leveled_codec:ledger_value_version() ) -> { leveled_codec:segment_hash(), @@ -2748,28 +2776,28 @@ check_notfound(CheckFrequency, CheckFun) -> %% @doc %% Prepare an object and its related key changes for addition to the Ledger %% via the Ledger Cache. -preparefor_ledgercache(?INKT_MPUT, ?DUMMY, SQN, _O, _S, {ObjSpecs, TTL}) -> - ObjChanges = leveled_codec:obj_objectspecs(ObjSpecs, SQN, TTL), +preparefor_ledgercache(?INKT_MPUT, ?DUMMY, SQN, _O, _S, {ObjSpecs, TTL}, VV) -> + ObjChanges = leveled_codec:obj_objectspecs(ObjSpecs, SQN, TTL, VV), {no_lookup, SQN, ObjChanges}; preparefor_ledgercache( - ?INKT_KEYD, LedgerKey, SQN, _Obj, _Size, {IdxSpecs, TTL} + ?INKT_KEYD, LedgerKey, SQN, _Obj, _Size, {IdxSpecs, TTL}, VV ) when LedgerKey =/= ?DUMMY -> {Bucket, Key} = leveled_codec:from_ledgerkey(LedgerKey), KeyChanges = - leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL), + leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL, VV), {no_lookup, SQN, KeyChanges}; preparefor_ledgercache( - _InkTag, LedgerKey, SQN, Obj, Size, {IdxSpecs, TTL} + _InkTag, LedgerKey, SQN, Obj, Size, {IdxSpecs, TTL}, VV ) when LedgerKey =/= ?DUMMY -> {Bucket, Key, MetaValue, {KeyH, _ObjH}, _LastMods} = - leveled_codec:generate_ledgerkv(LedgerKey, SQN, Obj, Size, TTL), + leveled_codec:generate_ledgerkv(LedgerKey, SQN, Obj, Size, TTL, VV), KeyChanges = [{LedgerKey, MetaValue}] ++ - leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL), + leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL, VV), {KeyH, SQN, KeyChanges}. -spec recalcfor_ledgercache( @@ -2780,7 +2808,8 @@ preparefor_ledgercache( integer(), leveled_codec:journal_keychanges(), ledger_cache(), - pid() + pid(), + leveled_codec:ledger_value_version() ) -> { leveled_codec:segment_hash(), @@ -2794,18 +2823,18 @@ preparefor_ledgercache( %% journal entry (i.e. KeyDeltas which may be a result of previously running %% with a retain strategy should be ignored). recalcfor_ledgercache( - InkTag, _LedgerKey, SQN, _Obj, _Size, {_IdxSpecs, _TTL}, _LC, _Pcl + InkTag, _LedgerKey, SQN, _Obj, _Size, {_IdxSpecs, _TTL}, _LC, _Pcl, _VV ) when InkTag == ?INKT_MPUT; InkTag == ?INKT_KEYD -> {no_lookup, SQN, []}; recalcfor_ledgercache( - _InkTag, LK, SQN, Obj, Size, {_Ignore, TTL}, LedgerCache, Penciller + _InkTag, LK, SQN, Obj, Size, {_Ignore, TTL}, LedgerCache, Penciller, VV ) when LK =/= ?DUMMY -> {Bucket, Key, MetaValue, {KeyH, _ObjH}, _LastMods} = - leveled_codec:generate_ledgerkv(LK, SQN, Obj, Size, TTL), + leveled_codec:generate_ledgerkv(LK, SQN, Obj, Size, TTL, VV), OldObject = case check_in_ledgercache(LK, KeyH, LedgerCache, loader) of false -> @@ -2818,21 +2847,21 @@ recalcfor_ledgercache( not_present -> not_present; {LK, LV} -> - case leveled_codec:get_metadata(LV) of - MDO when is_tuple(MDO) -> + case leveled_codec:ledgermd_sqnumd(LV) of + {_OSQN, MDO} when is_tuple(MDO) -> MDO end end, UpdMetadata = - case leveled_codec:get_metadata(MetaValue) of - MDU when is_tuple(MDU) -> + case leveled_codec:ledgermd_sqnumd(MetaValue) of + {_USQN, MDU} when is_tuple(MDU) -> MDU end, IdxSpecs = leveled_head:diff_indexspecs(element(1, LK), UpdMetadata, OldMetadata), {KeyH, SQN, [{LK, MetaValue}] ++ - leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL)}. + leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL, VV)}. -spec addto_ledgercache( { @@ -2924,7 +2953,15 @@ maybepush_ledgercache( Tab = Cache#ledger_cache.mem, CacheSize = ets:info(Tab, size), leveled_monitor:add_stat(Monitor, {ledger_cache_size_update, CacheSize}), - TimeToPush = maybe_withjitter(CacheSize, MaxCacheSize, MaxCacheMult), + TimeToPush = + maybe_withjitter( + CacheSize, + MaxCacheSize, + MaxCacheMult, + Cache#ledger_cache.max_sqn, + Cache#ledger_cache.min_sqn, + ?MAX_SQN_MULTIPLE * MaxCacheSize + ), if TimeToPush -> CacheToLoad = @@ -2948,17 +2985,29 @@ maybepush_ledgercache( end. -spec maybe_withjitter( - non_neg_integer(), pos_integer(), pos_integer() + non_neg_integer(), + pos_integer(), + pos_integer(), + non_neg_integer(), + non_neg_integer() | infinity, + pos_integer() ) -> boolean(). %% @doc %% Push down randomly, but the closer to 4 * the maximum size, the more likely %% a push should be maybe_withjitter( - CacheSize, MaxCacheSize, MaxCacheMult + CacheSize, MaxCacheSize, MaxCacheMult, _MaxSQN, _MinSQN, _MaxSQNDiff ) when CacheSize > MaxCacheSize -> R = rand:uniform(MaxCacheMult * MaxCacheSize), (CacheSize - MaxCacheSize) > R; -maybe_withjitter(_CacheSize, _MaxCacheSize, _MaxCacheMult) -> +maybe_withjitter( + _CacheSize, _MaxCacheSize, MaxCacheMult, MaxSQN, MinSQN, MaxSQNDiff +) when is_integer(MinSQN), (MaxSQN - MinSQN) > MaxSQNDiff -> + R = rand:uniform(MaxCacheMult * MaxSQNDiff), + ((MaxSQN - MinSQN) - MaxSQNDiff) > R; +maybe_withjitter( + _CacheSize, _MaxCacheSize, _MaxCacheMult, _MaxSQN, _MinSQN, _MaxSQNDiff +) -> false. -spec get_loadfun() -> initial_loadfun(). diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index a6761dec..6ae66877 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -13,16 +13,18 @@ -ifdef(TEST). -export([convert_to_ledgerv/5]). +-export([ledgermd_statuslmd/1, ledgermd_umd/1, create_v3_value/5]). -endif. -export([ inker_reload_strategy/1, - strip_to_seqonly/1, - strip_to_statusonly/1, - strip_to_segmentonly/1, - strip_to_keyseqonly/1, - strip_to_indexdetails/1, - striphead_to_v1details/1, + ledgermd_sqn/1, + ledgermd_status/1, + ledgermd_seg/1, + ledgermd_seglmd/1, + ledgermd_statussqn/1, + ledgermd_statussqnumd/1, + ledgermd_sqnumd/1, endkey_passed/2, key_dominates/2, to_objectkey/3, @@ -45,15 +47,14 @@ create_value_for_journal/3, revert_value_from_journal/1, revert_value_from_journal/2, - generate_ledgerkv/5, + generate_ledgerkv/6, get_size/2, get_keyandobjhash/2, - idx_indexspecs/5, - obj_objectspecs/3, + idx_indexspecs/6, + obj_objectspecs/4, segment_hash/1, next_key/1, return_proxy/4, - get_metadata/1, maybe_accumulate/5, accumulate_index/2, count_tombs/2 @@ -71,7 +72,7 @@ pos_integer(). -type segment_hash() :: % hash of the key to an aae segment - to be used in ledger filters - {integer(), integer()} | no_lookup. + {non_neg_integer(), non_neg_integer()} | no_lookup. -type head_value() :: any(). -type metadata() :: % null for empty metadata @@ -83,6 +84,7 @@ integer() | undefined. -type lastmod_range() :: {integer(), pos_integer() | infinity}. +-type ledger_value_version() :: 2 | 3. -type ledger_status() :: tomb | {active, non_neg_integer() | infinity}. -type primary_key() :: @@ -97,11 +99,12 @@ -type slimmed_key() :: {binary(), binary() | null} | binary() | null | all. -type ledger_value() :: - ledger_value_v1() | ledger_value_v2(). + ledger_value_v1() | ledger_value_v2() | ledger_value_v3(). -type ledger_value_v1() :: {sqn(), ledger_status(), segment_hash(), metadata()}. -type ledger_value_v2() :: {sqn(), ledger_status(), segment_hash(), metadata(), last_moddate()}. +-type ledger_value_v3() :: binary(). -type ledger_kv() :: {object_key(), ledger_value()}. -type compaction_method() :: @@ -180,6 +183,7 @@ query_key/0, ledger_key/0, ledger_value/0, + ledger_value_version/0, ledger_kv/0, compaction_strategy/0, compaction_method/0, @@ -248,35 +252,136 @@ headkey_to_canonicalbinary( %% @doc %% Some helper functions to get a sub_components of the key/value - --spec strip_to_statusonly(ledger_kv()) -> ledger_status(). -strip_to_statusonly({_, V}) -> element(2, V). - --spec strip_to_seqonly(ledger_kv()) -> non_neg_integer(). -strip_to_seqonly({_, V}) -> element(1, V). - --spec strip_to_segmentonly(ledger_kv()) -> segment_hash(). -strip_to_segmentonly({_LK, LV}) -> element(3, LV). - --spec strip_to_keyseqonly(ledger_kv()) -> {ledger_key(), integer()}. -strip_to_keyseqonly({LK, V}) -> {LK, element(1, V)}. - --spec strip_to_indexdetails(ledger_kv()) -> - {integer(), segment_hash(), last_moddate()}. -strip_to_indexdetails({_, {SQN, _, SegmentHash, _}}) -> - % A v1 value - {SQN, SegmentHash, undefined}; -strip_to_indexdetails({_, {SQN, _, SegmentHash, _, LMD}}) -> - % A v2 value should have a fith element - Last Modified Date - {SQN, SegmentHash, LMD}. - --spec striphead_to_v1details(ledger_value()) -> ledger_value(). -striphead_to_v1details(V) -> - {element(1, V), element(2, V), element(3, V), element(4, V)}. - --spec get_metadata(ledger_value()) -> metadata(). -get_metadata(LV) -> - element(4, LV). +% tomb | {active, non_neg_integer() | infinity} + +-spec ledgermd_status(ledger_value()) -> ledger_status(). +ledgermd_status(<<3:8/integer, 0:8/integer, _Rest/binary>>) -> + {active, infinity}; +ledgermd_status(<<3:8/integer, 1:4/integer, 0:4/integer, _Rest/binary>>) -> + tomb; +ledgermd_status(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [status]); +ledgermd_status(V) when is_tuple(V) -> + element(2, V). + +-spec ledgermd_sqn(ledger_value()) -> non_neg_integer(). +ledgermd_sqn( + << + 3:8/integer, + _:4/integer, + 0:4/integer, + 0:8/integer, + _:6/binary, + 4:8/integer, + _:4/binary, + Rem/binary + >> +) -> + % Short circuit the value extraction when object has no TTL, has a standard + % hash and the LMD is stored in 4 bytes (will be 22nd century before LMD is + % 5 bytes) + read_v3_value(Rem, sqn, [sqn], []); +ledgermd_sqn(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [sqn]); +ledgermd_sqn(V) when is_tuple(V) -> + element(1, V). + +-spec ledgermd_seg(ledger_value()) -> segment_hash(). +ledgermd_seg(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [seg_hash]); +ledgermd_seg(V) when is_tuple(V) -> + element(3, V). + +-spec ledgermd_statussqn(ledger_value()) -> + {ledger_status(), non_neg_integer()}. +ledgermd_statussqn( + << + 3:8/integer, + 0:8/integer, + 0:8/integer, + _:6/binary, + 4:8/integer, + _:4/binary, + Rem/binary + >> +) -> + % Short circuit the value extraction when object has no TTL, has a standard + % hash and the LMD is stored in 4 bytes (will be 22nd century before LMD is + % 5 bytes) + {{active, infinity}, read_v3_value(Rem, sqn, [sqn], [])}; +ledgermd_statussqn(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [status, sqn]); +ledgermd_statussqn(V) when is_tuple(V) -> + {element(2, V), element(1, V)}. + +-spec ledgermd_seglmd(ledger_value()) -> {segment_hash(), last_moddate()}. +ledgermd_seglmd( + << + 3:8/integer, + _:4/integer, + 0:4/integer, + 1:8/integer, + 0:8/integer, + _Rest/binary + >> +) -> + % Short circuit the value extraction when object is an index entry, so no + % hash, with no TTL + {no_lookup, undefined}; +ledgermd_seglmd(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [seg_hash, lmd]); +ledgermd_seglmd({_, _, SegHash, _, LMD}) -> + {SegHash, LMD}; +ledgermd_seglmd({_, _, SegHash, _}) -> + {SegHash, undefined}. + +-spec ledgermd_statuslmd(ledger_value()) -> {ledger_status(), last_moddate()}. +ledgermd_statuslmd(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [status, lmd]); +ledgermd_statuslmd({_, Status, _, _, LMD}) -> + {Status, LMD}; +ledgermd_statuslmd({_, Status, _, _}) -> + {Status, undefined}. + +-spec ledgermd_statussqnumd( + ledger_value() +) -> + {ledger_status(), non_neg_integer(), metadata() | null}. +ledgermd_statussqnumd(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [status, sqn, umd]); +ledgermd_statussqnumd(V) when is_tuple(V) -> + {element(2, V), element(1, V), element(4, V)}. + +-spec ledgermd_sqnumd( + ledger_value() +) -> + {non_neg_integer(), metadata() | null}. +ledgermd_sqnumd( + << + 3:8/integer, + _:4/integer, + 0:4/integer, + 0:8/integer, + _:6/binary, + 4:8/integer, + _:4/binary, + Rem/binary + >> +) -> + % Short circuit the value extraction when object has no TTL, has a standard + % hash and the LMD is stored in 4 bytes (will be 22nd century before LMD is + % 5 bytes) + read_v3_value(Rem, sqn, [sqn, umd], []); +ledgermd_sqnumd(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [sqn, umd]); +ledgermd_sqnumd(V) when is_tuple(V) -> + {element(1, V), element(4, V)}. + +-spec ledgermd_umd(ledger_value()) -> metadata() | null. +ledgermd_umd(<<3:8/integer, V/binary>>) -> + read_v3_value(V, [umd]); +ledgermd_umd(V) when is_tuple(V) -> + element(4, V). -spec maybe_accumulate( list(leveled_codec:ledger_kv()), @@ -293,59 +398,27 @@ get_metadata(LV) -> maybe_accumulate([], Acc, Count, _Filter, _Fun) -> {Acc, Count}; maybe_accumulate( - [{K, {_SQN, {active, TS}, _SH, _MD, undefined} = V} | T], - Acc, - Count, - {Now, _ModRange} = Filter, - AccFun -) when - TS >= Now --> - maybe_accumulate(T, AccFun(K, V, Acc), Count + 1, Filter, AccFun); -maybe_accumulate( - [{K, {_SQN, {active, TS}, _SH, _MD} = V} | T], - Acc, - Count, - {Now, _ModRange} = Filter, - AccFun + [{K, V} | T], Acc, Count, {Now, ModRange} = Filter, AccFun ) when - TS >= Now + ModRange == ?OPEN_LASTMOD_RANGE -> - maybe_accumulate(T, AccFun(K, V, Acc), Count + 1, Filter, AccFun); -maybe_accumulate( - [{_K, {_SQN, tomb, _SH, _MD, _LMD}} | T], - Acc, - Count, - Filter, - AccFun -) -> - maybe_accumulate(T, Acc, Count, Filter, AccFun); -maybe_accumulate( - [{_K, {_SQN, tomb, _SH, _MD}} | T], - Acc, - Count, - Filter, - AccFun -) -> - maybe_accumulate(T, Acc, Count, Filter, AccFun); -maybe_accumulate( - [{K, {_SQN, {active, TS}, _SH, _MD, LMD} = V} | T], - Acc, - Count, - {Now, {LowDate, HighDate}} = Filter, - AccFun -) when - TS >= Now, LMD >= LowDate, LMD =< HighDate --> - maybe_accumulate(T, AccFun(K, V, Acc), Count + 1, Filter, AccFun); -maybe_accumulate( - [_LV | T], - Acc, - Count, - Filter, - AccFun -) -> - maybe_accumulate(T, Acc, Count, Filter, AccFun). + case {ledgermd_status(V), Now} of + {{active, TS}, Now} when TS >= Now -> + maybe_accumulate(T, AccFun(K, V, Acc), Count + 1, Filter, AccFun); + _ -> + maybe_accumulate(T, Acc, Count, Filter, AccFun) + end; +maybe_accumulate([{K, V} | T], Acc, Count, Filter, AccFun) -> + case {ledgermd_statuslmd(V), Filter} of + {{{active, TS}, undefined}, {Now, _ModRange}} when TS >= Now -> + maybe_accumulate(T, AccFun(K, V, Acc), Count + 1, Filter, AccFun); + {{{active, TS}, LMD}, {Now, {LowDate, HighDate}}} when + TS >= Now, LMD >= LowDate, LMD =< HighDate + -> + maybe_accumulate(T, AccFun(K, V, Acc), Count + 1, Filter, AccFun); + _ -> + maybe_accumulate(T, Acc, Count, Filter, AccFun) + end. -spec accumulate_index( {boolean() | binary(), term_expression()}, @@ -430,7 +503,7 @@ check_captured_terms( %% When comparing two keys in the ledger need to find if one key comes before %% the other, or if the match, which key is "better" and should be the winner key_dominates(LObj, RObj) -> - strip_to_seqonly(LObj) >= strip_to_seqonly(RObj). + ledgermd_sqn(element(2, LObj)) >= ledgermd_sqn(element(2, RObj)). -spec count_tombs( list(ledger_kv()), non_neg_integer() @@ -438,8 +511,8 @@ key_dominates(LObj, RObj) -> non_neg_integer(). count_tombs([], Count) -> Count; -count_tombs([{_K, V} | T], Count) when is_tuple(V) -> - case element(2, V) of +count_tombs([{_K, V} | T], Count) -> + case ledgermd_status(V) of tomb -> count_tombs(T, Count + 1); _ -> @@ -810,13 +883,18 @@ is_full_journalentry(_OtherJKType) -> %%% Other Ledger Functions %%%============================================================================ --spec obj_objectspecs(list(tuple()), integer(), integer() | infinity) -> +-spec obj_objectspecs( + list(tuple()), + integer(), + integer() | infinity, + ledger_value_version() +) -> list(ledger_kv()). %% @doc %% Convert object specs to KV entries ready for the ledger -obj_objectspecs(ObjectSpecs, SQN, TTL) -> +obj_objectspecs(ObjectSpecs, SQN, TTL, VV) -> lists:map( - fun(ObjectSpec) -> gen_headspec(ObjectSpec, SQN, TTL) end, + fun(ObjectSpec) -> gen_headspec(ObjectSpec, SQN, TTL, VV) end, ObjectSpecs ). @@ -825,34 +903,45 @@ obj_objectspecs(ObjectSpecs, SQN, TTL) -> any(), any(), integer(), - integer() | infinity + integer() | infinity, + ledger_value_version() ) -> list(ledger_kv()). %% @doc %% Convert index specs to KV entries ready for the ledger -idx_indexspecs(IndexSpecs, Bucket, Key, SQN, TTL) -> +idx_indexspecs(IndexSpecs, Bucket, Key, SQN, TTL, VV) -> lists:map( fun({IdxOp, IdxFld, IdxTrm}) -> - gen_indexspec(Bucket, Key, IdxOp, IdxFld, IdxTrm, SQN, TTL) + gen_indexspec(Bucket, Key, IdxOp, IdxFld, IdxTrm, SQN, TTL, VV) end, IndexSpecs ). -gen_indexspec(Bucket, Key, IdxOp, IdxField, IdxTerm, SQN, TTL) -> +gen_indexspec(Bucket, Key, IdxOp, IdxField, IdxTerm, SQN, TTL, VV) -> Status = set_status(IdxOp, TTL), { to_objectkey(Bucket, Key, ?IDX_TAG, IdxField, IdxTerm), - {SQN, Status, no_lookup, null} + case VV of + 2 -> + {SQN, Status, no_lookup, null}; + 3 -> + create_v3_value(SQN, Status, no_lookup, null, undefined) + end }. --spec gen_headspec(object_spec(), integer(), integer() | infinity) -> +-spec gen_headspec( + object_spec(), + integer(), + integer() | infinity, + ledger_value_version() +) -> ledger_kv(). %% @doc %% Take an object_spec as passed in a book_mput, and convert it into to a %% valid ledger key and value. Supports different shaped tuples for different %% versions of the object_spec gen_headspec( - {IdxOp, v1, Bucket, Key, SubKey, LMD, Value}, SQN, TTL + {IdxOp, v1, Bucket, Key, SubKey, LMD, Value}, SQN, TTL, VV ) when is_binary(Key) -> @@ -865,13 +954,28 @@ gen_headspec( SKB when is_binary(SKB) -> to_objectkey(Bucket, {Key, SKB}, ?HEAD_TAG) end, - {K, {SQN, Status, segment_hash(K), Value, get_last_lastmodification(LMD)}}; + SegHash = segment_hash(K), + LMTS = get_last_lastmodification(LMD), + { + K, + case VV of + 2 -> + {SQN, Status, SegHash, Value, LMTS}; + 3 -> + create_v3_value(SQN, Status, SegHash, Value, LMTS) + end + }; gen_headspec( - {IdxOp, Bucket, Key, SubKey, Value}, SQN, TTL + {IdxOp, Bucket, Key, SubKey, Value}, SQN, TTL, VV ) when is_binary(Key) -> - gen_headspec({IdxOp, v1, Bucket, Key, SubKey, undefined, Value}, SQN, TTL). + gen_headspec( + {IdxOp, v1, Bucket, Key, SubKey, undefined, Value}, + SQN, + TTL, + VV + ). -spec return_proxy( leveled_head:object_tag(), @@ -908,12 +1012,13 @@ set_status(remove, _TTL) -> integer(), dynamic(), integer(), - non_neg_integer() | infinity + non_neg_integer() | infinity, + ledger_value_version() ) -> { key(), single_key(), - ledger_value_v2(), + ledger_value_v1() | ledger_value_v2() | ledger_value_v3(), {segment_hash(), non_neg_integer() | null}, list(erlang:timestamp()) }. @@ -929,7 +1034,7 @@ set_status(remove, _TTL) -> %% of the value to be used for equality checking between objects %% LastMods - the last modified dates for the object (may be multiple due to %% siblings) -generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> +generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS, VV) -> {Tag, Bucket, Key, _} = PrimaryKey, Status = case Obj of @@ -939,16 +1044,185 @@ generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> Hash = segment_hash(PrimaryKey), {MD, LastMods} = leveled_head:extract_metadata(Tag, Size, Obj), ObjHash = leveled_head:get_hash(Tag, MD), + LMD = get_last_lastmodification(LastMods), Value = - { - SQN, - Status, - Hash, - MD, - get_last_lastmodification(LastMods) - }, + case VV of + 1 -> + % To be used in testing to recerate old objects + {SQN, Status, Hash, MD}; + 2 -> + {SQN, Status, Hash, MD, LMD}; + 3 -> + create_v3_value(SQN, Status, Hash, MD, LMD) + end, {Bucket, Key, Value, {Hash, ObjHash}, LastMods}. +-spec create_v3_value( + non_neg_integer(), + ledger_status(), + segment_hash(), + leveled_head:object_metadata() | metadata(), + pos_integer() | undefined +) -> + binary(). +create_v3_value(SQN, Status, Hash, MD, LMTS) -> + SQNB = binary:encode_unsigned(SQN), + SQNBin = <<(byte_size(SQNB)):8/integer, SQNB/binary>>, + StatusBin = + case Status of + {active, infinity} -> + <<0:4/integer, 0:4/integer>>; + tomb -> + <<1:4/integer, 0:4/integer>>; + {active, TS} when is_integer(TS) -> + TSB = binary:encode_unsigned(TS), + <<2:4/integer, (byte_size(TSB)):4/integer, TSB/binary>> + end, + SegHashBin = + case Hash of + {SegHash, ExtraHash} -> + <<0:8/integer, SegHash:16/integer, ExtraHash:32/integer>>; + no_lookup -> + <<1:8/integer>> + end, + + MDBin = + case MD of + null -> + <<0:8/integer>>; + _ -> + case term_to_binary(MD) of + MDB -> + MDBSize = byte_size(MDB), + LengthByteSize = size_bytelength(MDBSize, 0), + << + 1:4/integer, + LengthByteSize:4/integer, + MDBSize:(LengthByteSize * 8)/integer, + MDB/binary + >> + end + end, + LMTSBin = + case LMTS of + LMTS when is_integer(LMTS) -> + LMTSB = binary:encode_unsigned(LMTS), + <<(byte_size(LMTSB)):8/integer, LMTSB/binary>>; + undefined -> + <<0:8/integer>> + end, + << + 3:8/integer, + StatusBin/binary, + SegHashBin/binary, + LMTSBin/binary, + SQNBin/binary, + MDBin/binary + >>. + +%% @doc How many bytes are required to store the length of the object +%% if the byte-size od the object is Size. e.g. Size <= 255 bytes has a length +%% of 1 byte, < 64KB a length of 2 bytes, < 16MB a length of 3 bytes etc. +%% This length will then be stored in 4-bits within the header of the item. +-spec size_bytelength(non_neg_integer(), 0..14) -> 1..15. +size_bytelength(Size, Acc) when Acc < 15 -> + case Size bsr 8 of + 0 -> + Acc + 1; + UpdSize -> + size_bytelength(UpdSize, Acc + 1) + end. + +read_v3_value(ValueBin, Items) -> + read_v3_value(ValueBin, status, Items, []). + +read_v3_value(_RemBin, _NextITem, [], [SingleItem]) -> + SingleItem; +read_v3_value(_RemBin, _NextItem, [], Acc) -> + list_to_tuple(lists:reverse(Acc)); +read_v3_value( + <>, status, [Next | Items], Acc +) when I < 2 -> + case {Next, I} of + {Next, _I} when Next =/= status -> + read_v3_value(Rem, seg_hash, [Next | Items], Acc); + {status, 0} -> + read_v3_value(Rem, seg_hash, Items, [{active, infinity} | Acc]); + {status, 1} -> + read_v3_value(Rem, seg_hash, Items, [tomb | Acc]) + end; +read_v3_value( + <<2:4/integer, L:4/integer, Rem/binary>>, status, [Next | Items], Acc +) -> + <> = Rem, + case Next of + status -> + read_v3_value(Rest, seg_hash, Items, [ + {active, binary:decode_unsigned(TS)} | Acc + ]); + _ -> + read_v3_value(Rest, seg_hash, [Next | Items], Acc) + end; +read_v3_value(<<1:8/integer, Rem/binary>>, seg_hash, [Next | Items], Acc) -> + case Next of + seg_hash -> + read_v3_value(Rem, lmd, Items, [no_lookup | Acc]); + _ -> + read_v3_value(Rem, lmd, [Next | Items], Acc) + end; +read_v3_value( + <<0:8/integer, SH:16/integer, EH:32/integer, Rem/binary>>, + seg_hash, + [Next | Items], + Acc +) -> + case Next of + seg_hash -> + read_v3_value(Rem, lmd, Items, [{SH, EH} | Acc]); + _ -> + read_v3_value(Rem, lmd, [Next | Items], Acc) + end; +read_v3_value(<<0:8/integer, Rem/binary>>, lmd, [Next | Items], Acc) -> + case Next of + lmd -> + read_v3_value(Rem, sqn, Items, [undefined | Acc]); + _ -> + read_v3_value(Rem, sqn, [Next | Items], Acc) + end; +read_v3_value(<>, lmd, [Next | Items], Acc) -> + <> = Rem, + case Next of + lmd -> + read_v3_value(Rest, sqn, Items, [binary:decode_unsigned(LMD) | Acc]); + _ -> + read_v3_value(Rest, sqn, [Next | Items], Acc) + end; +read_v3_value(<>, sqn, [Next | Items], Acc) -> + <> = Rem, + case Next of + sqn -> + read_v3_value(Rest, umd, Items, [ + binary:decode_unsigned(SQN) | Acc + ]); + _ -> + read_v3_value(Rest, umd, [Next | Items], Acc) + end; +read_v3_value(<<0:8/integer, Rem/binary>>, umd, [umd], Acc) -> + read_v3_value(Rem, umd, [], [null | Acc]); +read_v3_value( + << + 1:4/integer, + L:4/integer, + UmdSize:(L * 8)/integer, + Rem/binary + >>, + umd, + [umd], + Acc +) -> + <> = Rem, + read_v3_value(Rest, umd, [], [binary_to_term(UMD) | Acc]). + -spec get_last_lastmodification( list(erlang:timestamp()) | undefined ) -> pos_integer() | undefined. @@ -965,8 +1239,10 @@ get_last_lastmodification(LastMods) -> get_size(PK, Value) -> {Tag, _Bucket, _Key, _} = PK, - MD = element(4, Value), - leveled_head:get_size(Tag, MD). + case ledgermd_umd(Value) of + MD when is_tuple(MD) -> + leveled_head:get_size(Tag, MD) + end. -spec get_keyandobjhash(tuple(), tuple()) -> tuple(). %% @doc @@ -975,13 +1251,15 @@ get_size(PK, Value) -> %% the sorted vclock) get_keyandobjhash(LK, Value) -> {Tag, Bucket, Key, _} = LK, - MD = element(4, Value), case Tag of ?IDX_TAG -> % returns {Bucket, Key, IdxValue} from_ledgerkey(LK); _ -> - {Bucket, Key, leveled_head:get_hash(Tag, MD)} + case ledgermd_umd(Value) of + MD when is_tuple(MD) -> + {Bucket, Key, leveled_head:get_hash(Tag, MD)} + end end. -spec next_key(key()) -> key(). @@ -1011,9 +1289,26 @@ next_key({Type, Bucket}) when is_binary(Type), is_binary(Bucket) -> ) -> leveled_codec:ledger_value(). convert_to_ledgerv(PK, SQN, Obj, Size, TS) -> {_B, _K, MV, _H, _LMs} = - leveled_codec:generate_ledgerkv(PK, SQN, Obj, Size, TS), + leveled_codec:generate_ledgerkv(PK, SQN, Obj, Size, TS, 2), MV. +accumulate_legacy_object_test() -> + LK = + to_objectkey(<<"Bucket1">>, <<"Key1">>, o), + Chunk = crypto:strong_rand_bytes(64), + {_, _, LV, _, _} = generate_ledgerkv(LK, 100, Chunk, 64, infinity, 1), + Fun = fun(K, V, Acc) -> [{K, V} | Acc] end, + {Acc, C} = + maybe_accumulate( + [{LK, LV}], + [], + 0, + {leveled_util:integer_now(), {0, 10}}, + Fun + ), + ?assertMatch(1, C), + ?assertMatch([{LK, LV}], Acc). + valid_ledgerkey_test() -> UserDefTag = {user_defined, <<"B">>, <<"K">>, null}, ?assertMatch(true, isvalid_ledgerkey(UserDefTag)), @@ -1031,7 +1326,7 @@ indexspecs_test() -> {add, "t1_bin", "adbc123"}, {remove, "t1_bin", "abdc456"} ], - Changes = idx_indexspecs(IndexSpecs, "Bucket", "Key2", 1, infinity), + Changes = idx_indexspecs(IndexSpecs, "Bucket", "Key2", 1, infinity, 2), ?assertMatch( { {i, "Bucket", {"t1_int", 456}, "Key2"}, @@ -1091,6 +1386,34 @@ headspec_v0v1_test() -> V1 = {add, v1, <<"B">>, <<"K">>, <<"SK">>, undefined, {<<"V">>}}, V0 = {add, <<"B">>, <<"K">>, <<"SK">>, {<<"V">>}}, TTL = infinity, - ?assertMatch(true, gen_headspec(V0, 1, TTL) == gen_headspec(V1, 1, TTL)). + ?assertMatch( + true, + gen_headspec(V0, 1, TTL, 2) == gen_headspec(V1, 1, TTL, 2) + ). + +v3_value_test() -> + SQN = 1000, + Status = {active, infinity}, + Hash = segment_hash(<<"K">>), + UMD = {<<"Bin1">>, <<"Bin2">>, erlang:phash2(<<"Bin2">>), 1024}, + LMD = leveled_util:integer_now(), + V3Val = create_v3_value(SQN, Status, Hash, UMD, LMD), + ?assertMatch(SQN, ledgermd_sqn(V3Val)), + ?assertMatch(Status, ledgermd_status(V3Val)), + ?assertMatch({Status, SQN}, ledgermd_statussqn(V3Val)), + ?assertMatch({Hash, LMD}, ledgermd_seglmd(V3Val)), + ?assertMatch({Status, SQN, UMD}, ledgermd_statussqnumd(V3Val)), + ?assertMatch({Status, LMD}, ledgermd_statuslmd(V3Val)), + ?assertMatch(UMD, ledgermd_umd(V3Val)), + + TempStatus = {active, leveled_util:integer_now() + 100}, + V3ValB = create_v3_value(SQN, TempStatus, Hash, UMD, LMD), + ?assertMatch(SQN, ledgermd_sqn(V3ValB)), + ?assertMatch(TempStatus, ledgermd_status(V3ValB)), + ?assertMatch({TempStatus, SQN}, ledgermd_statussqn(V3ValB)), + ?assertMatch({Hash, LMD}, ledgermd_seglmd(V3ValB)), + ?assertMatch({TempStatus, SQN, UMD}, ledgermd_statussqnumd(V3ValB)), + ?assertMatch({TempStatus, LMD}, ledgermd_statuslmd(V3ValB)), + ?assertMatch(UMD, ledgermd_umd(V3ValB)). -endif. diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 197ea21d..35427a4c 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -51,7 +51,7 @@ -record(state, { owner :: pid() | undefined, root_path :: string() | undefined, - pending_deletions = dict:new() :: dict:dict(), + pending_deletions = maps:new() :: map(), sst_options :: sst_options() }). @@ -137,7 +137,7 @@ handle_cast( -> {ManifestSQN, Deletions} = handle_work(Work, RP, State#state.sst_options, PCL), - PDs = dict:store(ManifestSQN, Deletions, State#state.pending_deletions), + PDs = maps:put(ManifestSQN, Deletions, State#state.pending_deletions), ?STD_LOG(pc022, [ManifestSQN]), {noreply, State#state{pending_deletions = PDs}, ?MIN_TIMEOUT}; handle_cast( @@ -526,16 +526,16 @@ grooming_scorer(HighestTC, BestME, [ME | MEs]) -> end. return_deletions(ManifestSQN, PendingDeletionD) -> - % The returning of deletions had been seperated out as a failure to fetch - % here had caased crashes of the clerk. The root cause of the failure to + % The returning of deletions had been separated out as a failure to fetch + % here had caused crashes of the clerk. The root cause of the failure to % fetch was the same clerk being asked to do the same work twice - and this % should be blocked now by the ongoing_work boolean in the Penciller % LoopData % % So this is now allowed to crash again - PendingDeletions = dict:fetch(ManifestSQN, PendingDeletionD), + PendingDeletions = maps:get(ManifestSQN, PendingDeletionD), ?STD_LOG(pc021, [ManifestSQN]), - {PendingDeletions, dict:erase(ManifestSQN, PendingDeletionD)}. + {PendingDeletions, maps:remove(ManifestSQN, PendingDeletionD)}. %%%============================================================================ %%% Test diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 745e05bd..1af9e2f0 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -770,7 +770,7 @@ handle_call( true -> LedgerTable; false -> - leveled_tree:from_orderedset(LedgerTable, ?CACHE_TYPE) + leveled_tree:from_ets(LedgerTable, ?CACHE_TYPE) end, case leveled_pmem:add_to_cache( @@ -877,7 +877,7 @@ handle_call( fun(LKV) -> CheckSeg = leveled_sst:extract_hash( - leveled_codec:strip_to_segmentonly(LKV) + leveled_codec:ledgermd_seg(element(2, LKV)) ), case CheckSeg of CheckSeg when @@ -1863,7 +1863,7 @@ compare_to_sqn(ObjSQN, _SQN) when is_integer(ObjSQN) -> % confusion in snapshots. current; compare_to_sqn(Obj, SQN) -> - compare_to_sqn(leveled_codec:strip_to_seqonly(Obj), SQN). + compare_to_sqn(leveled_codec:ledgermd_sqn(element(2, Obj)), SQN). -spec maybelog_fetch_timing( leveled_monitor:monitor(), @@ -2264,7 +2264,7 @@ maybe_pause_push(PCL, KL) -> lists:foldl( fun({K, V}, {AccSL, AccIdx, MinSQN, MaxSQN}) -> UpdSL = [{K, V} | AccSL], - SQN = leveled_codec:strip_to_seqonly({K, V}), + SQN = leveled_codec:ledgermd_sqn(V), H = leveled_codec:segment_hash(K), UpdIdx = leveled_pmem:prepare_for_index(AccIdx, H), {UpdSL, UpdIdx, min(SQN, MinSQN), max(SQN, MaxSQN)} @@ -2894,14 +2894,14 @@ foldwithimm_simple_test() -> ], IMM2 = leveled_tree:from_orderedlist(lists:ukeysort(1, KL1A), ?CACHE_TYPE), IMMiter = - leveled_tree:match_range( + leveled_tree:between( {o, <<"Bucket1">>, <<"Key1">>, null}, {o, null, null, null}, IMM2 ), AccFun = fun(K, V, Acc) -> - SQN = leveled_codec:strip_to_seqonly({K, V}), + SQN = leveled_codec:ledgermd_sqn(V), Acc ++ [{K, SQN}] end, Acc = @@ -2954,7 +2954,7 @@ foldwithimm_simple_test() -> KL1B = [AddKV | KL1A], IMM3 = leveled_tree:from_orderedlist(lists:ukeysort(1, KL1B), ?CACHE_TYPE), IMMiterB = - leveled_tree:match_range( + leveled_tree:between( {o, <<"Bucket1">>, <<"Key1">>, null}, {o, null, null, null}, IMM3 diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index c2af2ce6..8e4c71d8 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -114,7 +114,7 @@ manifest_sqn = 0 :: non_neg_integer(), % The current manifest SQN snapshots = [] :: list(snapshot()), - % A list of snaphots (i.e. clones) + % A list of snapshots (i.e. clones) min_snapshot_sqn = 0 :: integer(), % The smallest snapshot manifest SQN in the snapshot list pending_deletes = new_pending_deletions() :: pending_deletions(), @@ -127,7 +127,7 @@ start_key :: leveled_codec:object_key(), end_key :: leveled_codec:object_key(), owner :: pid(), - filename :: string(), + filename :: file:filename(), bloom = none :: leveled_ebloom:bloom() | none }). @@ -137,8 +137,9 @@ -type manifest_entry() :: #manifest_entry{}. -type manifest_owner() :: pid(). -type lsm_level() :: 0..7. --type pending_deletions() :: dict:dict(). --type blooms() :: dict:dict(). +-type pending_deletions() :: + #{file:filename() => {non_neg_integer(), manifest_entry()}}. +-type blooms() :: map(). -type selector_strategy() :: random | {grooming, fun((list(manifest_entry())) -> manifest_entry())}. @@ -230,7 +231,7 @@ load_manifest(Manifest, LoadFun, SQNFun) -> UpdLevels = array:set(LevelIdx, L1, AccMan#manifest.levels), FoldBloomFun = fun({P, B}, BAcc) -> - dict:store(P, B, BAcc) + maps:put(P, B, BAcc) end, UpdBlooms = lists:foldl(FoldBloomFun, AccMan#manifest.blooms, LvlBloom), @@ -263,10 +264,10 @@ close_manifest(Manifest, CloseEntryFun) -> lists:foreach(CloseLevelFun, lists:seq(0, Manifest#manifest.basement)), ClosePDFun = - fun({_FN, {_SQN, ME}}) -> + fun(_FN, {_SQN, ME}) -> CloseEntryFun(ME) end, - lists:foreach(ClosePDFun, dict:to_list(Manifest#manifest.pending_deletes)). + maps:foreach(ClosePDFun, Manifest#manifest.pending_deletes). -spec save_manifest(manifest(), string()) -> ok. %% @doc @@ -478,7 +479,7 @@ remove_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> manifest(), integer(), integer(), list() | manifest_entry() ) -> manifest(). %% @doc -%% Switch a manifest etry from this level to the level below (i.e when there +%% Switch a manifest entry from this level to the level below (i.e when there %% are no overlapping manifest entries in the level below) switch_manifest_entry(Manifest, ManSQN, SrcLevel, Entry) -> % Move to level below - so needs to be removed but not marked as a @@ -695,12 +696,13 @@ release_snapshot(Manifest, Pid) -> %% remove the file from the manifest's list of pending_deletes. -spec ready_to_delete(manifest(), string()) -> boolean(). ready_to_delete(Manifest, Filename) -> - PendingDelete = dict:find(Filename, Manifest#manifest.pending_deletes), + PendingDelete = + maps:get(Filename, Manifest#manifest.pending_deletes, undefined), case {PendingDelete, Manifest#manifest.min_snapshot_sqn} of - {{ok, _}, 0} -> + {Found, 0} when Found =/= undefined -> % no shapshots true; - {{ok, {ChangeSQN, _ME}}, N} when N >= ChangeSQN -> + {{ChangeSQN, _ME}, N} when N >= ChangeSQN -> % Every snapshot is looking at a version of history after this % was removed true; @@ -716,7 +718,7 @@ clear_pending(Manifest, [], true) -> clear_pending(Manifest, [], false) -> Manifest; clear_pending(Manifest, [FN | RestFN], MaybeRelease) -> - PDs = dict:erase(FN, Manifest#manifest.pending_deletes), + PDs = maps:remove(FN, Manifest#manifest.pending_deletes), clear_pending( Manifest#manifest{pending_deletes = PDs}, RestFN, @@ -773,8 +775,8 @@ levelzero_present(Manifest) -> %% Check to see if a hash is present in a manifest entry by using the exported %% bloom filter check_bloom(Manifest, FP, Hash) -> - case dict:find(FP, Manifest#manifest.blooms) of - {ok, Bloom} when is_binary(Bloom) -> + case maps:get(FP, Manifest#manifest.blooms, undefined) of + Bloom when is_binary(Bloom) -> leveled_ebloom:check_hash(Hash, Bloom); _ -> true @@ -1033,7 +1035,7 @@ replace_entry(LevelIdx, Level, Removals, Additions) -> update_pendingdeletes(ManSQN, Removals, PendingDeletes) -> DelFun = fun(E, Acc) -> - dict:store(E#manifest_entry.filename, {ManSQN, E}, Acc) + maps:put(E#manifest_entry.filename, {ManSQN, E}, Acc) end, Entries = case is_list(Removals) of @@ -1070,11 +1072,11 @@ update_blooms(Removals, Additions, Blooms) -> RemFun = fun(R, BloomD) -> - dict:erase(R#manifest_entry.owner, BloomD) + maps:remove(R#manifest_entry.owner, BloomD) end, AddFun = fun(A, BloomD) -> - dict:store(A#manifest_entry.owner, A#manifest_entry.bloom, BloomD) + maps:put(A#manifest_entry.owner, A#manifest_entry.bloom, BloomD) end, StripFun = fun(A) -> @@ -1100,15 +1102,16 @@ key_lookup_level(LevelIdx, [Entry | Rest], Key) when LevelIdx =< 1 -> key_lookup_level(LevelIdx, Rest, Key) end; key_lookup_level(_LevelIdx, Level, Key) -> - StartKeyFun = - fun(ME) -> - ME#manifest_entry.start_key - end, - case leveled_tree:search(Key, Level, StartKeyFun) of + case leveled_tree:search(Key, Level) of none -> false; {_EK, ME} -> - ME#manifest_entry.owner + case Key >= ME#manifest_entry.start_key of + true -> + ME#manifest_entry.owner; + false -> + false + end end. range_lookup_int(Manifest, LevelIdx, StartKey, EndKey, MakePointerFun) -> @@ -1141,11 +1144,14 @@ range_lookup_level(LevelIdx, Level, QStartKey, QEndKey) when LevelIdx =< 1 -> {In, _After} = lists:splitwith(NotAfterFun, MaybeIn), In; range_lookup_level(_LevelIdx, Level, QStartKey, QEndKey) -> - StartKeyFun = - fun(ME) -> - ME#manifest_entry.start_key + EndRangeFun = + fun(ER, _FirstRHSKey, FirstRHSME) -> + not leveled_codec:endkey_passed( + ER, + FirstRHSME#manifest_entry.start_key + ) end, - Range = leveled_tree:search_range(QStartKey, QEndKey, Level, StartKeyFun), + Range = leveled_tree:between(QStartKey, QEndKey, Level, EndRangeFun), MapFun = fun({_EK, ME}) -> ME @@ -1200,9 +1206,9 @@ seconds_now() -> {MegaNow, SecNow, _} = os:timestamp(), MegaNow * 1000000 + SecNow. -new_blooms() -> dict:new(). +new_blooms() -> maps:new(). -new_pending_deletions() -> dict:new(). +new_pending_deletions() -> maps:new(). %%%============================================================================ %%% Test diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index 10a82b77..81525bda 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -204,7 +204,7 @@ check_levelzero(Key, Hash, PosList, TreeList) -> merge_trees(StartKey, EndKey, TreeList, LevelMinus1) -> lists:foldl( fun(Tree, Acc) -> - R = leveled_tree:match_range(StartKey, EndKey, Tree), + R = leveled_tree:between(StartKey, EndKey, Tree), lists:ukeymerge(1, Acc, R) end, [], diff --git a/src/leveled_runner.erl b/src/leveled_runner.erl index 57cef9f3..de01d983 100644 --- a/src/leveled_runner.erl +++ b/src/leveled_runner.erl @@ -149,7 +149,7 @@ bucket_list(SnapFun, Tag, FoldBucketsFun, InitAcc, MaxBuckets) -> %% This has the special capability that it will expect a message to be thrown %% during the query - and handle this without crashing the penciller snapshot %% This allows for this query to be used with a max_results check in the -%% applictaion - and to throw a stop message to be caught by the worker +%% application - and to throw a stop message to be caught by the worker %% handling the runner. This behaviour will not prevent the snapshot from %% closing neatly, allowing delete_pending files to be cleared without waiting %% for a timeout @@ -722,8 +722,7 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) -> % a fold_objects), then a metadata object needs to be built to be % returned - but a quick check that Key is present in the Journal % is made first - {SQN, _St, _MH, MD} = - leveled_codec:striphead_to_v1details(V), + {SQN, MD} = leveled_codec:ledgermd_sqnumd(V), {B, K} = case leveled_codec:from_ledgerkey(LK) of {B0, K0} -> @@ -766,8 +765,8 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) -> AccFun. check_presence(Key, Value, InkerClone) -> - {LedgerKey, SQN} = leveled_codec:strip_to_keyseqonly({Key, Value}), - case leveled_inker:ink_keycheck(InkerClone, LedgerKey, SQN) of + SQN = leveled_codec:ledgermd_sqn(Value), + case leveled_inker:ink_keycheck(InkerClone, Key, SQN) of probably -> true; missing -> diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 40c60d58..fed1ef69 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1327,7 +1327,7 @@ segment_checker(false) -> sqn_only(not_present) -> not_present; sqn_only(KV) -> - leveled_codec:strip_to_seqonly(KV). + leveled_codec:ledgermd_sqn(element(2, KV)). -spec extract_hash( leveled_codec:segment_hash() @@ -2111,20 +2111,12 @@ term_prefix_filter(N, Prefix) -> end. lookup_slot(Key, Tree, FilterFun) -> - StartKeyFun = - fun(_V) -> - all - end, % The penciller should never ask for presence out of range - so will - % always return a slot (as we don't compare to StartKey) - {_LK, Slot} = leveled_tree:search(FilterFun(Key), Tree, StartKeyFun), + % always return a slot + {_LK, Slot} = leveled_tree:search(FilterFun(Key), Tree), Slot. lookup_slots(StartKey, EndKey, Tree, FilterFun) -> - StartKeyFun = - fun(_V) -> - all - end, MapFun = fun({_LK, Slot}) -> Slot @@ -2140,11 +2132,11 @@ lookup_slots(StartKey, EndKey, Tree, FilterFun) -> _ -> FilterFun(EndKey) end, SlotList = - leveled_tree:search_range( + leveled_tree:between( FilteredStartKey, FilteredEndKey, Tree, - StartKeyFun + fun(_, _, _) -> true end ), {EK, _EndSlot} = lists:last(SlotList), { @@ -2199,7 +2191,7 @@ finalise_posbin({_, {PosBin, HashAcc, LMDAcc}}) -> non_neg_integer(), position_acc() }) -> {non_neg_integer(), position_acc()}. accumulate_position(NextKV, {NHC, {PosBin, HashAcc, LMDAcc}}) -> - {_SQN, H1, LMD} = leveled_codec:strip_to_indexdetails(NextKV), + {H1, LMD} = leveled_codec:ledgermd_seglmd(element(2, NextKV)), LMDAcc0 = take_max_lastmoddate(LMD, LMDAcc), case extract_hash(H1) of PosH1 when is_integer(PosH1) -> @@ -3655,7 +3647,7 @@ maybe_expand_keys(KVL) -> maybe_reap_expiredkey(KV, {false, _}) -> KV; maybe_reap_expiredkey(KV, {true, CurrTS}) -> - case leveled_codec:strip_to_statusonly(KV) of + case leveled_codec:ledgermd_status(element(2, KV)) of {_, TS} when is_integer(TS), CurrTS > TS -> none; tomb -> @@ -3847,6 +3839,7 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> ), Chunk = crypto:strong_rand_bytes(64), MV = leveled_codec:convert_to_ledgerv(LK, Seqn, Chunk, 64, infinity), + is_tuple(MV) orelse error(bad_type), MD = element(4, MV), is_tuple(MD) orelse error(bad_type), ?assertMatch(undefined, element(3, MD)), @@ -3876,7 +3869,8 @@ generate_indexkey(Term, Count) -> "Bucket", "Key" ++ integer_to_list(Count), Count, - infinity + infinity, + 2 ). append_performance_test_() -> @@ -4577,11 +4571,7 @@ simple_persisted_rangesegfilter_tester(SSTNewFun) -> fun(LK) -> case lists:keyfind(LK, 1, KVList1) of LKV when LKV =/= false -> - extract_hash( - leveled_codec:strip_to_segmentonly( - LKV - ) - ) + extract_hash(leveled_codec:ledgermd_seg(element(2, LKV))) end end, SegList = @@ -4836,7 +4826,7 @@ reader_hibernate_tester() -> {ok, Pid, {FirstKey, LastKey}, _Bloom} = testsst_new(RP, Filename, 1, KVList1, length(KVList1), {0, native}), ?assertMatch({FirstKey, FV}, sst_get(Pid, FirstKey)), - SQN = leveled_codec:strip_to_seqonly({FirstKey, FV}), + SQN = leveled_codec:ledgermd_sqn(FV), ?assertMatch( SQN, sst_getsqn(Pid, FirstKey, leveled_codec:segment_hash(FirstKey)) @@ -5192,7 +5182,7 @@ hashmatching_bytreesize_test() -> null}, LKV = leveled_codec:generate_ledgerkv( - LK, X, V, byte_size(V), infinity + LK, X, V, byte_size(V), infinity, 2 ), {_Bucket, _Key, MetaValue, _Hashes, _LastMods} = LKV, {LK, MetaValue} @@ -6029,7 +6019,8 @@ single_key_test() -> <<"Bucket">>, <<"Key">>, 1, - infinity + infinity, + 2 ), {ok, P2, {IdxK, IdxK}, _Bloom2} = sst_new(?TEST_AREA, FileName, 1, [{IdxK, IdxV}], 6000, OptsSST), @@ -6107,7 +6098,7 @@ strange_range_test() -> GenerateValue = fun(K) -> element( - 3, leveled_codec:generate_ledgerkv(K, 1, V, 16, infinity) + 3, leveled_codec:generate_ledgerkv(K, 1, V, 16, infinity, 2) ) end, @@ -6139,7 +6130,8 @@ strange_range_test() -> <<"Bucket">>, <<"Key">>, 1, - infinity + infinity, + 2 ), {ok, P2, {_FIdxK, _EIdxK}, _Bloom2} = sst_new( @@ -6201,7 +6193,7 @@ blocks_required_test() -> element( 3, leveled_codec:generate_ledgerkv( - StdKey(I), I, Chunk, 32, infinity + StdKey(I), I, Chunk, 32, infinity, 2 ) ) end, @@ -6210,7 +6202,7 @@ blocks_required_test() -> element( 3, leveled_codec:generate_ledgerkv( - IdxKey(I), I, <<>>, 0, infinity + IdxKey(I), I, <<>>, 0, infinity, 2 ) ) end, diff --git a/src/leveled_sstblock.erl b/src/leveled_sstblock.erl index 5003db2b..7a23fa7d 100644 --- a/src/leveled_sstblock.erl +++ b/src/leveled_sstblock.erl @@ -783,7 +783,7 @@ v1_block_tester(Lookup, BlockMethod, BlockSize, SibMetaBin, B) -> null}, LKV = leveled_codec:generate_ledgerkv( - LK, X, V, byte_size(V), infinity + LK, X, V, byte_size(V), infinity, 2 ), {_Bucket, _Key, MetaValue, _Hashes, _LastMods} = LKV, {LK, MetaValue} diff --git a/src/leveled_tree.erl b/src/leveled_tree.erl index c6dd92c1..b4877269 100644 --- a/src/leveled_tree.erl +++ b/src/leveled_tree.erl @@ -1,35 +1,57 @@ %% -------- TREE --------- %% -%% This module is intended to address two issues -%% - the lack of iterator_from support in OTP16 gb_trees -%% - the time to convert from/to list in gb_trees +%% There are two trees supported within leveled, tree and idxt - however idxt +%% is the only one used at present. %% -%% Leveled had had a skiplist implementation previously, and this is a -%% variation on that. The Treein this case is a bunch of sublists of length -%% SKIP_WIDTH with the start_keys in a gb_tree. +%% The tree type is a gb_trees:tree(), and the idxt is a variation whereby 1 in +%% 12 keys go into a gb_trees:tree(), and there is a tuple of sublists 12-wide +%% kept to one side. All operations in idxt are a tree walk followed by a +%% check against the relevant sublists. +%% +%% The indexed keys in idxt are the end keys in each sublist. +%% +%% The idxt approach is faster to convert to/from a list. It is also faster +%% to find matching ranges. It is slower at lookups, but only marginally +%% slower. +%% +%% There are timing tests within the eunit test suite for this module to +%% demonstrate the difference. +%% +%% This is not a general purpose solution. The `between` function used the +%% leveled_codec:endkey_passed/2 function to determine the top of the range. +%% This function will act in an expected way (e.g. out of range if +%% RangeEndKey < TreeKey) if not a tuple, but differently if a key-like tuple +%% from the leveled_codec (due to the need to handle a null within the tuple +%% in the expected way). +%% +%% A more efficient implementation would be possible if simple term ordering +%% was used instead. Do not use this as a generic alternative to gb_trees +%% because of this inefficiency. -module(leveled_tree). --include("leveled.hrl"). - -export([ from_orderedlist/2, - from_orderedset/2, + from_ets/2, from_orderedlist/3, - from_orderedset/3, + from_ets/3, to_list/1, - match_range/3, - search_range/4, + between/3, + between/4, match/2, - search/3, + search/2, tsize/1, empty/1 ]). --define(SKIP_WIDTH, 16). +-define(SKIP_WIDTH, 12). --type tree_type() :: tree | idxt | skpl. --type leveled_tree() :: {tree_type(), integer(), any()}. +-type tree_type() :: tree | idxt. +-type leveled_tree_tree() :: {tree, gb_trees:tree()}. +-type leveled_tree_idxt() :: + {idxt, non_neg_integer(), {tuple(), gb_trees:tree()}}. +-type leveled_tree() :: + leveled_tree_tree() | leveled_tree_idxt(). -export_type([leveled_tree/0]). @@ -37,23 +59,22 @@ %%% API %%%============================================================================ --spec from_orderedset(ets:tab(), tree_type()) -> leveled_tree(). +-spec from_ets(ets:tab(), tree_type()) -> leveled_tree(). %% @doc %% Convert an ETS table of Keys and Values (of table type ordered_set) into a %% leveled_tree of the given type. -from_orderedset(Table, Type) -> - from_orderedlist(ets:tab2list(Table), Type, ?SKIP_WIDTH). +from_ets(Table, Type) -> + from_ets(Table, Type, ?SKIP_WIDTH). --spec from_orderedset( - ets:tab(), tree_type(), integer() | auto +-spec from_ets( + ets:tab(), tree_type(), pos_integer() ) -> leveled_tree(). %% @doc %% Convert an ETS table of Keys and Values (of table type ordered_set) into a %% leveled_tree of the given type. The SkipWidth is an integer representing %% the underlying list size joined in the tree (the trees are all trees of -%% lists of this size). For the skpl type the width can be auto-sized based -%% on the length -from_orderedset(Table, Type, SkipWidth) -> +%% lists of this size). +from_ets(Table, Type, SkipWidth) when is_integer(SkipWidth), SkipWidth > 0 -> from_orderedlist(ets:tab2list(Table), Type, SkipWidth). -spec from_orderedlist(list(tuple()), tree_type()) -> leveled_tree(). @@ -70,193 +91,108 @@ from_orderedlist(OrderedList, Type) -> %% Convert a list of Keys and Values (of table type ordered_set) into a %% leveled_tree of the given type. The SkipWidth is an integer representing %% the underlying list size joined in the tree (the trees are all trees of -%% lists of this size). For the skpl type the width can be auto-sized based -%% on the length -from_orderedlist(OrderedList, tree, SkipWidth) -> - L = length(OrderedList), - {tree, L, tree_fromorderedlist(OrderedList, [], L, SkipWidth)}; +%% lists of this size). +from_orderedlist(OrderedList, tree, _SkipWidth) -> + {tree, gb_trees:from_orddict(OrderedList)}; from_orderedlist(OrderedList, idxt, SkipWidth) -> L = length(OrderedList), - {idxt, L, idxt_fromorderedlist(OrderedList, {[], [], 1}, L, SkipWidth)}; -from_orderedlist(OrderedList, skpl, _SkipWidth) -> - L = length(OrderedList), - SkipWidth = - % Autosize the skip width - case L of - L when L > 4096 -> 32; - L when L > 512 -> 16; - L when L > 64 -> 8; - _ -> 4 - end, - {skpl, L, skpl_fromorderedlist(OrderedList, L, SkipWidth, 2)}. + {idxt, L, idxt_fromorderedlist(OrderedList, {[], [], 1}, L, SkipWidth)}. --spec match(tuple() | integer(), leveled_tree()) -> none | {value, any()}. +-spec match(tuple(), leveled_tree()) -> none | {value, any()}. %% @doc %% Return the value from a tree associated with an exact match for the given %% key. This assumes the tree contains the actual keys and values to be -%% macthed against, not a manifest representing ranges of keys and values. -match(Key, {tree, _L, Tree}) -> - Iter = tree_iterator_from(Key, Tree), - case tree_next(Iter) of - none -> - none; - {_NK, SL, _Iter} -> - lookup_match(Key, SL) - end; +%% matched against, not a manifest representing ranges of keys and values. +match(Key, {tree, Tree}) -> + gb_trees:lookup(Key, Tree); match(Key, {idxt, _L, {TLI, IDX}}) when is_tuple(TLI) -> - Iter = tree_iterator_from(Key, IDX), - case tree_next(Iter) of + Iter = gb_trees:iterator_from(Key, IDX), + case gb_trees:next(Iter) of none -> none; - {_NK, ListID, _Iter} -> + {_NK, ListID, _Iter} when is_integer(ListID) -> lookup_match(Key, element(ListID, TLI)) - end; -match(Key, {skpl, _L, SkipList}) -> - SL0 = skpl_getsublist(Key, SkipList), - lookup_match(Key, SL0). + end. --spec search( - tuple() | integer(), - leveled_tree(), - fun((leveled_pmanifest:manifest_entry()) -> leveled_codec:object_key()) -) -> - none | tuple(). +-spec search(tuple(), leveled_tree()) -> none | tuple(). %% @doc -%% Search is used when the tree is a manifest of key ranges and it is necessary -%% to find a rnage which may contain the key. The StartKeyFun is used if the -%% values contain extra information that can be used to determine if the key is -%% or is not present. -search(Key, {tree, _L, Tree}, StartKeyFun) -> - Iter = tree_iterator_from(Key, Tree), - case tree_next(Iter) of +%% Find the first key >= to the SearchKey in the tree. +search(Key, {tree, Tree}) -> + Iter = gb_trees:iterator_from(Key, Tree), + case gb_trees:next(Iter) of none -> none; - {_NK, SL, _Iter} -> - {K, V} = lookup_best(Key, SL), - case Key < StartKeyFun(V) of - true -> - none; - false -> - {K, V} - end + {NK, V, _Iter} -> + {NK, V} end; -search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) when is_tuple(TLI) -> - Iter = tree_iterator_from(Key, IDX), - case tree_next(Iter) of +search(Key, {idxt, _L, {TLI, IDX}}) when is_tuple(TLI) -> + Iter = gb_trees:iterator_from(Key, IDX), + case gb_trees:next(Iter) of none -> none; - {_NK, ListID, _Iter} -> - {K, V} = lookup_best(Key, element(ListID, TLI)), - case Key < StartKeyFun(V) of - true -> - none; - false -> - {K, V} - end - end; -search(Key, {skpl, _L, SkipList}, StartKeyFun) -> - SL0 = skpl_getsublist(Key, SkipList), - case lookup_best(Key, SL0) of - {K, V} -> - case Key < StartKeyFun(V) of - true -> - none; - false -> - {K, V} - end; - none -> - none + {_NK, ListID, _Iter} when is_integer(ListID) -> + lookup_best(Key, element(ListID, TLI)) end. --spec match_range( - tuple() | integer() | all, - tuple() | integer() | all, +-spec between( + tuple() | all, + tuple() | all, leveled_tree() ) -> list(). %% @doc -%% Return a range of value between trees from a tree associated with an -%% exact match for the given key. This assumes the tree contains the actual -%% keys and values to be macthed against, not a manifest representing ranges -%% of keys and values. -%% -%% The keyword all can be used as a substitute for the StartKey to remove a -%% constraint from the range. -match_range(StartRange, EndRange, Tree) -> +%% Return a range of {K, V} pairs from the tree between the StartRange key +%% and the EndRange key. +between(StartRange, EndRange, Tree) -> EndRangeFun = fun(ER, FirstRHSKey, _FirstRHSValue) -> ER == FirstRHSKey end, - match_range(StartRange, EndRange, Tree, EndRangeFun). + between(StartRange, EndRange, Tree, EndRangeFun). --spec match_range( - tuple() | integer() | all, - tuple() | integer() | all, +-spec between( + tuple() | all, + tuple() | all, leveled_tree(), fun((term(), term(), term()) -> boolean()) ) -> list(). %% @doc -%% As match_range/3 but a function can be passed to be used when comparing the -%5 EndKey with a key in the tree (such as leveled_codec:endkey_passed), where -%% Erlang term comparison will not give the desired result. -match_range(StartRange, EndRange, {tree, _L, Tree}, EndRangeFun) -> - treelookup_range_start(StartRange, EndRange, Tree, EndRangeFun); -match_range(StartRange, EndRange, {idxt, _L, Tree}, EndRangeFun) -> - idxtlookup_range_start(StartRange, EndRange, Tree, EndRangeFun); -match_range(StartRange, EndRange, {skpl, _L, SkipList}, EndRangeFun) -> - skpllookup_to_range(StartRange, EndRange, SkipList, EndRangeFun). - --spec search_range( - tuple() | integer() | all, - tuple() | integer() | all, - leveled_tree(), - fun((leveled_pmanifest:manifest_entry()) -> leveled_codec:object_key()) -) -> - list(). -%% @doc -%% Extract a range from a tree, with search used when the tree is a manifest -%% of key ranges and it is necessary to find a rnage which may encapsulate the -%% key range. +%% As between/3 but a function can be passed to be used when comparing the +%% EndKey with a key in the tree. %% -%% The StartKeyFun is used if the values contain extra information that can be -%% used to determine if the key is or is not present. -search_range(StartRange, EndRange, Tree, StartKeyFun) -> - EndRangeFun = - fun(ER, _FirstRHSKey, FirstRHSValue) -> - StartRHSKey = StartKeyFun(FirstRHSValue), - not leveled_codec:endkey_passed(ER, StartRHSKey) - end, - case Tree of - {tree, _L, T} -> - treelookup_range_start(StartRange, EndRange, T, EndRangeFun); - {idxt, _L, T} -> - idxtlookup_range_start(StartRange, EndRange, T, EndRangeFun); - {skpl, _L, SL} -> - skpllookup_to_range(StartRange, EndRange, SL, EndRangeFun) - end. +%% The EndKey is the next key in the tree which is not strictly less than the +%% EndRange Key. +%% +%% e.g. To always include this key: +%% EndRangeFun = fun(_, _, _) -> true end +%% To only include if it is equal: +%% EndRangeFun = fun(ER, EK, _EV) -> ER == EK end +%% +%% The value of this final key is also passed in the function, should the entry +%% represent a range of entries, and the value includes the start of that +%% range. For examples of using this see leveled_pmanifest. +%% +%% The top of the range is checked using leveled_codec:endkey_passed/2, which +%% is different to strict erlang term order then the keys are a tuple in the +%% format expected in the leveled_codec module. This function has special +%% handling of null elements within the tuple. +between(StartRange, EndRange, {tree, Tree}, EndRangeFun) -> + treelookup_range_start(StartRange, EndRange, Tree, EndRangeFun); +between(StartRange, EndRange, {idxt, _L, Tree}, EndRangeFun) -> + idxtlookup_range_start(StartRange, EndRange, Tree, EndRangeFun). -spec to_list(leveled_tree()) -> list(). %% @doc %% Collapse the tree back to a list -to_list({tree, _L, Tree}) -> - FoldFun = - fun({_MK, SL}, Acc) -> - Acc ++ SL - end, - lists:foldl(FoldFun, [], tree_to_list(Tree)); +to_list({tree, Tree}) -> + gb_trees:to_list(Tree); to_list({idxt, _L, {TLI, _IDX}}) when is_tuple(TLI) -> - lists:append(tuple_to_list(TLI)); -to_list({skpl, _L, SkipList}) when is_list(SkipList) -> - FoldFun = - fun({_M, SL}, Acc) -> - [SL | Acc] - end, - Lv1List = lists:reverse(lists:foldl(FoldFun, [], SkipList)), - Lv0List = lists:reverse(lists:foldl(FoldFun, [], lists:append(Lv1List))), - lists:append(Lv0List). + lists:append(tuple_to_list(TLI)). -spec tsize(leveled_tree()) -> integer(). %% @doc %% Return the count of items in a tree +tsize({tree, Tree}) -> + gb_trees:size(Tree); tsize({_Type, L, _Tree}) -> L. @@ -264,24 +200,14 @@ tsize({_Type, L, _Tree}) -> %% @doc %% Return an empty tree of the given type empty(tree) -> - {tree, 0, empty_tree()}; + {tree, gb_trees:empty()}; empty(idxt) -> - {idxt, 0, {{}, empty_tree()}}; -empty(skpl) -> - {skpl, 0, []}. + {idxt, 0, {{}, gb_trees:empty()}}. %%%============================================================================ %%% Internal Functions %%%============================================================================ -tree_fromorderedlist([], TmpList, _L, _SkipWidth) -> - gb_trees:from_orddict(lists:reverse(TmpList)); -tree_fromorderedlist(OrdList, TmpList, L, SkipWidth) -> - SubLL = min(SkipWidth, L), - {Head, Tail} = lists:split(SubLL, OrdList), - {LastK, _LastV} = lists:last(Head), - tree_fromorderedlist(Tail, [{LastK, Head} | TmpList], L - SubLL, SkipWidth). - idxt_fromorderedlist([], {TmpListElements, TmpListIdx, _C}, _L, _SkipWidth) -> { list_to_tuple(lists:reverse(TmpListElements)), @@ -298,29 +224,6 @@ idxt_fromorderedlist(OrdList, {TmpListElements, TmpListIdx, C}, L, SkipWidth) -> SkipWidth ). -skpl_fromorderedlist(SkipList, _L, _SkipWidth, 0) -> - SkipList; -skpl_fromorderedlist(SkipList, L, SkipWidth, Height) -> - SkipList0 = roll_list(SkipList, L, [], SkipWidth), - skpl_fromorderedlist(SkipList0, length(SkipList0), SkipWidth, Height - 1). - -roll_list([], 0, SkipList, _SkipWidth) -> - lists:reverse(SkipList); -roll_list(KVList, L, SkipList, SkipWidth) -> - SubLL = min(SkipWidth, L), - {Head, Tail} = lists:split(SubLL, KVList), - {LastK, _LastV} = lists:last(Head), - roll_list(Tail, L - SubLL, [{LastK, Head} | SkipList], SkipWidth). - -% lookup_match(_Key, []) -> -% none; -% lookup_match(Key, [{EK, _EV}|_Tail]) when EK > Key -> -% none; -% lookup_match(Key, [{Key, EV}|_Tail]) -> -% {value, EV}; -% lookup_match(Key, [_Top|Tail]) -> -% lookup_match(Key, Tail). - lookup_match(Key, KVList) -> case lists:keyfind(Key, 1, KVList) of false -> @@ -329,237 +232,116 @@ lookup_match(Key, KVList) -> {value, Value} end. -lookup_best(_Key, []) -> - none; lookup_best(Key, [{EK, EV} | _Tail]) when EK >= Key -> {EK, EV}; lookup_best(Key, [_Top | Tail]) -> lookup_best(Key, Tail). treelookup_range_start(StartRange, EndRange, Tree, EndRangeFun) -> - Iter0 = tree_iterator_from(StartRange, Tree), - case tree_next(Iter0) of - none -> - []; - {NK, SL, Iter1} -> - PredFun = - fun({K, _V}) -> - K < StartRange - end, - {_LHS, RHS} = lists:splitwith(PredFun, SL), - treelookup_range_end(EndRange, {NK, RHS}, Iter1, [], EndRangeFun) - end. + Iter0 = gb_trees:iterator_from(StartRange, Tree), + lists:reverse(tree_range_fold(Iter0, EndRange, EndRangeFun, [])). -treelookup_range_end(EndRange, {NK0, SL0}, Iter0, Output, EndRangeFun) -> - PredFun = - fun({K, _V}) -> - not leveled_codec:endkey_passed(EndRange, K) - end, - case leveled_codec:endkey_passed(EndRange, NK0) of - true -> - {LHS, RHS} = lists:splitwith(PredFun, SL0), - [{FirstRHSKey, FirstRHSValue} | _Rest] = RHS, - case EndRangeFun(EndRange, FirstRHSKey, FirstRHSValue) of +tree_range_fold(Iter, EndRange, EndRangeFun, Acc) -> + case gb_trees:next(Iter) of + none -> + Acc; + {NK, NV, Iter1} -> + case leveled_codec:endkey_passed(EndRange, NK) of true -> - Output ++ LHS ++ [{FirstRHSKey, FirstRHSValue}]; + case EndRangeFun(EndRange, NK, NV) of + true -> + [{NK, NV} | Acc]; + false -> + Acc + end; false -> - Output ++ LHS - end; - false -> - UpdOutput = Output ++ SL0, - case tree_next(Iter0) of - none -> - UpdOutput; - {NK1, SL1, Iter1} -> - treelookup_range_end( - EndRange, - {NK1, SL1}, - Iter1, - UpdOutput, - EndRangeFun - ) + tree_range_fold(Iter1, EndRange, EndRangeFun, [ + {NK, NV} | Acc + ]) end end. idxtlookup_range_start(StartRange, EndRange, {TLI, IDX}, EndRangeFun) -> % TLI tuple of lists, IDS is a gb_tree of End Keys mapping to tuple % indexes - Iter0 = tree_iterator_from(StartRange, IDX), - case tree_next(Iter0) of + Iter0 = gb_trees:iterator_from(StartRange, IDX), + case gb_trees:next(Iter0) of none -> []; {NK, ListID, Iter1} -> - PredFun = + BeforeFun = fun({K, _V}) -> K < StartRange end, - {_LHS, RHS} = lists:splitwith(PredFun, element(ListID, TLI)), + {_LHS, RHS} = lists:splitwith(BeforeFun, element(ListID, TLI)), % The RHS is the list of {EK, SK} elements where the EK >= the % StartRange, otherwise the LHS falls before the range - idxtlookup_range_end( - EndRange, {TLI, NK, RHS}, Iter1, [], EndRangeFun - ) + case idxtlookup_range_end(EndRange, NK, Iter1, []) of + {[], true} -> + right_trim(RHS, EndRangeFun, EndRange); + {[], false} -> + RHS; + {[HdIdx | RestIdx], RTrim} -> + RHS ++ + lists:foldl( + fun(I, Acc) -> element(I, TLI) ++ Acc end, + case RTrim of + true -> + right_trim( + element(HdIdx, TLI), + EndRangeFun, + EndRange + ); + false -> + [] + end, + case RTrim of + true -> + RestIdx; + false -> + [HdIdx | RestIdx] + end + ) + end end. -idxtlookup_range_end(EndRange, {TLI, NK0, SL0}, Iter0, Output, EndRangeFun) -> +right_trim(SubList, EndRangeFun, EndRange) -> PredFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndRange, K) % true if EndRange is after K end, + {LHS, [{FirstRHSKey, FirstRHSValue} | _Rest]} = + lists:splitwith(PredFun, SubList), + case EndRangeFun(EndRange, FirstRHSKey, FirstRHSValue) of + true -> + % The start key is not after the end of the range + % and so this should be included in the range + LHS ++ [{FirstRHSKey, FirstRHSValue}]; + false -> + % the start key of the next key is after the end + % of the range and so should not be included + LHS + end. + +idxtlookup_range_end(EndRange, NK0, Iter0, Acc) -> case leveled_codec:endkey_passed(EndRange, NK0) of true -> - % The end key of this list is after the end of the range, so no - % longer interested in any of the rest of the tree - just this - % sublist - {LHS, RHS} = lists:splitwith(PredFun, SL0), - % Split the {EK, SK} pairs based on the EndRange. Note that the - % last key is passed the end range - so the RHS cannot be empty, it - % must at least include the last key (as NK0 is at the end of SL0). - [{FirstRHSKey, FirstRHSValue} | _Rest] = RHS, - case EndRangeFun(EndRange, FirstRHSKey, FirstRHSValue) of - true -> - % The start key is not after the end of the range - % and so this should be included in the range - Output ++ LHS ++ [{FirstRHSKey, FirstRHSValue}]; - false -> - % the start key of the next key is after the end - % of the range and so should not be included - Output ++ LHS - end; + {Acc, true}; false -> - UpdOutput = Output ++ SL0, - case tree_next(Iter0) of + case gb_trees:next(Iter0) of none -> - UpdOutput; + {Acc, false}; {NK1, ListID, Iter1} -> idxtlookup_range_end( EndRange, - {TLI, NK1, element(ListID, TLI)}, + NK1, Iter1, - UpdOutput, - EndRangeFun + [ListID | Acc] ) end end. -skplfold_range([], _StartRange, _EndRange, Acc) -> - Acc; -skplfold_range([{K, _SL} | Rest], StartRange, EndRange, Acc) when - StartRange > K --> - skplfold_range(Rest, StartRange, EndRange, Acc); -skplfold_range([{K, SL} | Rest], StartRange, EndRange, Acc) -> - case leveled_codec:endkey_passed(EndRange, K) of - true -> - [SL | Acc]; - false -> - skplfold_range(Rest, StartRange, EndRange, [SL | Acc]) - end. - -skpllookup_to_range(StartRange, EndRange, SkipList, EndRangeFun) -> - Lv1List = - lists:reverse( - skplfold_range(SkipList, StartRange, EndRange, []) - ), - Lv0List = - lists:reverse( - skplfold_range( - lists:append(Lv1List), StartRange, EndRange, [] - ) - ), - BeforeFun = - fun({K, _V}) -> - K < StartRange - end, - AfterFun = - fun({K, V}) -> - case leveled_codec:endkey_passed(EndRange, K) of - false -> - true; - true -> - EndRangeFun(EndRange, K, V) - end - end, - - case Lv0List of - [] -> - []; - [SingleList] -> - RHS = lists:dropwhile(BeforeFun, SingleList), - lists:takewhile(AfterFun, RHS); - [LHList, RHList] -> - RHSofLHL = lists:dropwhile(BeforeFun, LHList), - LHSofRHL = lists:takewhile(AfterFun, RHList), - RHSofLHL ++ LHSofRHL; - [LHL | Rest] -> - RHSofLHL = lists:dropwhile(BeforeFun, LHL), - LHSofRHL = lists:takewhile(AfterFun, lists:last(Rest)), - MidLists = lists:sublist(Rest, length(Rest) - 1), - lists:append([RHSofLHL] ++ MidLists ++ [LHSofRHL]) - end. - -skpl_getsublist(Key, SkipList) -> - FoldFun = - fun({Mark, SL}, Acc) -> - case {Acc, Mark} of - {[], Mark} when Mark >= Key -> - SL; - _ -> - Acc - end - end, - SL1 = lists:foldl(FoldFun, [], SkipList), - lists:foldl(FoldFun, [], SL1). - -%%%============================================================================ -%%% Balance tree implementation -%%%============================================================================ - -empty_tree() -> - gb_trees:empty(). - -tree_to_list(T) -> - gb_trees:to_list(T). - -tree_iterator_from(K, T) -> - % For OTP 16 compatibility with gb_trees - iterator_from(K, T). - -tree_next(I) -> - % For OTP 16 compatibility with gb_trees - next(I). - -iterator_from(S, {_, T}) -> - iterator_1_from(S, T). - -iterator_1_from(S, T) -> - iterator_from(S, T, []). - -iterator_from(S, {K, _, _, T}, As) when K < S -> - iterator_from(S, T, As); -iterator_from(_, {_, _, nil, _} = T, As) -> - [T | As]; -iterator_from(S, {_, _, L, _} = T, As) -> - iterator_from(S, L, [T | As]); -iterator_from(_, nil, As) -> - As. - -next([{X, V, _, T} | As]) -> - {X, V, iterator(T, As)}; -next([]) -> - none. - -%% The iterator structure is really just a list corresponding to -%% the call stack of an in-order traversal. This is quite fast. - -iterator({_, _, nil, _} = T, As) -> - [T | As]; -iterator({_, _, L, _} = T, As) -> - iterator(L, [T | As]); -iterator(nil, As) -> - As. - %%%============================================================================ %%% Test %%%============================================================================ @@ -625,30 +407,31 @@ idxt_search_test() -> search_test_by_type(idxt), extra_searchrange_test_by_type(idxt). -skpl_search_test() -> - search_test_by_type(skpl), - extra_searchrange_test_by_type(skpl). - search_test_by_type(Type) -> MapFun = fun(N) -> - {N * 4, N * 4 - 2} + {{N * 4}, {N * 4 - 2}} end, KL = lists:map(MapFun, lists:seq(1, 50)), T = from_orderedlist(KL, Type), + EndRangeFun = + fun(ER, _FirstRHSKey, FirstRHSValue) -> + ER >= FirstRHSValue + end, - StartKeyFun = fun(V) -> V end, statistics(runtime), - ?assertMatch([], search_range(0, 1, T, StartKeyFun)), - ?assertMatch([], search_range(201, 202, T, StartKeyFun)), - ?assertMatch([{4, 2}], search_range(2, 4, T, StartKeyFun)), - ?assertMatch([{4, 2}], search_range(2, 5, T, StartKeyFun)), - ?assertMatch([{4, 2}, {8, 6}], search_range(2, 6, T, StartKeyFun)), - ?assertMatch(50, length(search_range(2, 200, T, StartKeyFun))), - ?assertMatch(50, length(search_range(2, 198, T, StartKeyFun))), - ?assertMatch(49, length(search_range(2, 197, T, StartKeyFun))), - ?assertMatch(49, length(search_range(4, 197, T, StartKeyFun))), - ?assertMatch(48, length(search_range(5, 197, T, StartKeyFun))), + ?assertMatch([], between({0}, {1}, T, EndRangeFun)), + ?assertMatch([], between({201}, {202}, T, EndRangeFun)), + ?assertMatch([{{4}, {2}}], between({2}, {4}, T, EndRangeFun)), + ?assertMatch([{{4}, {2}}], between({2}, {5}, T, EndRangeFun)), + ?assertMatch([{{4}, {2}}, {{8}, {6}}], between({2}, {6}, T, EndRangeFun)), + ?assertMatch(50, length(between({2}, {200}, T, EndRangeFun))), + ?assertMatch(50, length(between({2}, {198}, T, EndRangeFun))), + ?assertMatch(50, length(between(all, {198}, T, EndRangeFun))), + ?assertMatch(49, length(between({2}, {197}, T, EndRangeFun))), + ?assertMatch(49, length(between(all, {197}, T, EndRangeFun))), + ?assertMatch(49, length(between({4}, {197}, T, EndRangeFun))), + ?assertMatch(48, length(between({5}, {197}, T, EndRangeFun))), {_, T1} = statistics(runtime), io:format( user, @@ -662,31 +445,22 @@ tree_oor_test() -> idxt_oor_test() -> outofrange_test_by_type(idxt). -skpl_oor_test() -> - outofrange_test_by_type(skpl). - outofrange_test_by_type(Type) -> MapFun = fun(N) -> - {N * 4, N * 4 - 2} + {{N * 4}, N * 4 - 2} end, KL = lists:map(MapFun, lists:seq(1, 50)), T = from_orderedlist(KL, Type), io:format("Out of range searches~n"), - ?assertMatch(none, match(0, T)), - ?assertMatch(none, match(5, T)), - ?assertMatch(none, match(97, T)), - ?assertMatch(none, match(197, T)), - ?assertMatch(none, match(201, T)), - - StartKeyFun = fun(V) -> V end, + ?assertMatch(none, match({0}, T)), + ?assertMatch(none, match({5}, T)), + ?assertMatch(none, match({97}, T)), + ?assertMatch(none, match({197}, T)), + ?assertMatch(none, match({201}, T)), - ?assertMatch(none, search(0, T, StartKeyFun)), - ?assertMatch(none, search(5, T, StartKeyFun)), - ?assertMatch(none, search(97, T, StartKeyFun)), - ?assertMatch(none, search(197, T, StartKeyFun)), - ?assertMatch(none, search(201, T, StartKeyFun)). + ?assertMatch(none, search({201}, T)). tree_tolist_test() -> tolist_test_by_type(tree). @@ -694,13 +468,10 @@ tree_tolist_test() -> idxt_tolist_test() -> tolist_test_by_type(idxt). -skpl_tolist_test() -> - tolist_test_by_type(skpl). - tolist_test_by_type(Type) -> MapFun = fun(N) -> - {N * 4, N * 4 - 2} + {{N * 4}, N * 4 - 2} end, KL = lists:map(MapFun, lists:seq(1, 50)), T = from_orderedlist(KL, Type), @@ -713,29 +484,21 @@ timing_tests_tree_test_() -> timing_tests_idxt_test_() -> {timeout, 60, fun idxt_timing/0}. -timing_tests_skpl_test_() -> - {timeout, 60, fun skpl_timing/0}. - tree_timing() -> - log_tree_test_by_(16, tree, 8000), - log_tree_test_by_(16, tree, 4000), - log_tree_test_by_(4, tree, 256). + log_tree_test_by_(1, tree, 8000), + log_tree_test_by_(1, tree, 4000), + log_tree_test_by_(1, tree, 2000), + log_tree_test_by_(1, tree, 256), + log_tree_test_by_simplekey_(1, tree, 256). idxt_timing() -> - log_tree_test_by_(16, idxt, 8000), - log_tree_test_by_(16, idxt, 4000), - log_tree_test_by_(4, idxt, 256), - log_tree_test_by_(16, idxt, 256), - log_tree_test_by_simplekey_(16, idxt, 256). - -skpl_timing() -> - log_tree_test_by_(auto, skpl, 8000), - log_tree_test_by_(auto, skpl, 4000), - log_tree_test_by_simplekey_(auto, skpl, 4000), - log_tree_test_by_(auto, skpl, 512), - log_tree_test_by_simplekey_(auto, skpl, 512), - log_tree_test_by_(auto, skpl, 256), - log_tree_test_by_simplekey_(auto, skpl, 256). + log_tree_test_by_(12, idxt, 8000), + log_tree_test_by_(12, idxt, 4000), + log_tree_test_by_(32, idxt, 2000), + log_tree_test_by_(12, idxt, 2000), + log_tree_test_by_(6, idxt, 2000), + log_tree_test_by_(12, idxt, 256), + log_tree_test_by_simplekey_(12, idxt, 256). log_tree_test_by_(Width, Type, N) -> KL = lists:ukeysort(1, generate_randomkeys(1, N, 1, N div 5)), @@ -770,7 +533,7 @@ tree_test_by_(Width, Type, KL, ComplexKey) -> OS = ets:new(test, [ordered_set, private]), ets:insert(OS, KL), SWaETS = os:timestamp(), - Tree0 = from_orderedset(OS, Type, Width), + Tree0 = from_ets(OS, Type, Width), io:format( user, "Generating tree from ETS in ~w microseconds" ++ @@ -855,8 +618,122 @@ tree_test_by_(Width, Type, KL, ComplexKey) -> user, "Search all keys twice for near match in ~w microseconds~n", [timer:now_diff(os:timestamp(), SWaSRCH2)] + ), + + BigRanges = + lists:map( + fun(I) -> + get_random_range( + KL, + case I rem 2 of + 0 -> exact; + 1 -> over + end, + 400 + ) + end, + lists:seq(1, 1000) + ), + ok = test_ranges(BigRanges, Tree0, Tree1, big), + MidRanges = + lists:map( + fun(I) -> + get_random_range( + KL, + case I rem 2 of + 0 -> exact; + 1 -> over + end, + 40 + ) + end, + lists:seq(1, 1000) + ), + ok = test_ranges(MidRanges, Tree0, Tree1, mid), + SmallRanges = + lists:map( + fun(I) -> + get_random_range( + KL, + case I rem 2 of + 0 -> exact; + 1 -> over + end, + 10 + ) + end, + lists:seq(1, 1000) + ), + ok = test_ranges(SmallRanges, Tree0, Tree1, small), + + {TC0, OL} = timer:tc(fun() -> to_list(Tree0) end), + {TC1, OL} = timer:tc(fun() -> to_list(Tree1) end), + + io:format(user, "Reverted both to_list in ~w microseconds~n", [TC0 + TC1]). + +test_ranges(TestRanges, Tree0, Tree1, Size) -> + {TCRange0, RL0} = + timer:tc( + fun() -> + lists:map( + fun({SK, EK, SL}) -> + {between(SK, EK, Tree0), SL} + end, + TestRanges + ) + end + ), + {TCRange1, RL1} = + timer:tc( + fun() -> + lists:map( + fun({SK, EK, SL}) -> + {between(SK, EK, Tree1), SL} + end, + TestRanges + ) + end + ), + lists:foreach( + fun({R, Exp}) -> + ?assertMatch(Exp, R) + end, + RL0 + ), + lists:foreach( + fun({R, Exp}) -> + ?assertMatch(Exp, R) + end, + RL1 + ), + io:format( + user, + "Matched 1000 ~w ranges in both trees in ~w microseconds~n", + [Size, TCRange0 + TCRange1] ). +get_random_range(KL, RangeType, MaxSize) -> + L = length(KL), + R = rand:uniform(L - 5), + RangeSize = min(max(4, rand:uniform(L - R)), MaxSize), + SL = lists:sublist(KL, R, RangeSize), + case {RangeType, lists:last(SL)} of + {exact, LastKV} -> + {element(1, hd(SL)), element(1, LastKV), SL}; + {over, {{o_kv, B, FullKey, null}, _LastV}} -> + LastKey = + { + o_kv, + B, + list_to_binary(binary_to_list(FullKey) ++ "0"), + null + }, + {element(1, hd(SL)), LastKey, SL}; + {over, {K, _V}} -> + LastKey = list_to_binary(binary_to_list(K) ++ "0"), + {element(1, hd(SL)), LastKey, SL} + end. + tree_matchrange_test() -> matchrange_test_by_type(tree), extra_matchrange_test_by_type(tree). @@ -865,10 +742,6 @@ idxt_matchrange_test() -> matchrange_test_by_type(idxt), extra_matchrange_test_by_type(idxt). -skpl_matchrange_test() -> - matchrange_test_by_type(skpl), - extra_matchrange_test_by_type(skpl). - matchrange_test_by_type(Type) -> N = 4000, KL = lists:ukeysort(1, generate_randomkeys(1, N, 1, N div 5)), @@ -892,7 +765,7 @@ matchrange_test_by_type(Type) -> LengthR = fun(SK, EK, T) -> - length(match_range(SK, EK, T)) + length(between(SK, EK, T)) end, KL_Length = length(KL), @@ -927,7 +800,7 @@ extra_matchrange_test_by_type(Type) -> {o_kv, SB, list_to_binary(binary_to_list(SK) ++ "0"), null}, ERangeK = {o_kv, EB, list_to_binary(binary_to_list(EK) ++ "0"), null}, - ?assertMatch(49, length(match_range(SRangeK, ERangeK, Tree0))) + ?assertMatch(49, length(between(SRangeK, ERangeK, Tree0))) end, lists:foreach(TestRangeLFun, RangeLists). @@ -940,7 +813,13 @@ extra_searchrange_test_by_type(Type) -> SubL = lists:sublist(KL, 2000, 3100), - SKFun = fun(V) -> V end, + EndRangeFun = + fun(ER, _FirstRHSKey, FirstRHSME) -> + not leveled_codec:endkey_passed( + ER, + FirstRHSME + ) + end, TestRangeLFun = fun(P) -> @@ -958,7 +837,7 @@ extra_searchrange_test_by_type(Type) -> BRangeK = {o_kv, EB, list_to_binary(binary_to_list(EK) ++ "0"), null}, ?assertMatch( - 25, length(search_range(FRangeK, BRangeK, Tree0, SKFun)) + 25, length(between(FRangeK, BRangeK, Tree0, EndRangeFun)) ) end, lists:foreach(TestRangeLFun, lists:seq(1, 50)). @@ -976,26 +855,29 @@ match_fun(Tree) -> end. search_exactmatch_fun(Tree) -> - StartKeyFun = fun(_V) -> all end, fun({K, V}) -> - ?assertMatch({K, V}, search(K, Tree, StartKeyFun)) + ?assertMatch({K, V}, search(K, Tree)) end. search_nearmatch_fun(Tree) -> - StartKeyFun = fun(_V) -> all end, fun({K, {NK, NV}}) -> - ?assertMatch({NK, NV}, search(K, Tree, StartKeyFun)) + ?assertMatch({NK, NV}, search(K, Tree)) end. empty_test() -> T0 = empty(tree), ?assertMatch(0, tsize(T0)), - T1 = empty(skpl), - ?assertMatch(0, tsize(T1)), T2 = empty(idxt), ?assertMatch(0, tsize(T2)). -search_range_idx_test() -> +between_idx_test() -> + EndRangeFun = + fun(ER, _FirstRHSKey, FirstRHSME) -> + not leveled_codec:endkey_passed( + ER, + leveled_pmanifest:entry_startkey(FirstRHSME) + ) + end, Tree = {idxt, 1, { {[ @@ -1010,14 +892,16 @@ search_range_idx_test() -> ) } ]}, - {1, {{o_rkv, <<"Bucket1">>, <<"Key1">>, null}, 1, nil, nil}} + gb_trees:from_orddict( + [{{o_rkv, <<"Bucket1">>, <<"Key1">>, null}, 1}] + ) }}, R = - search_range( + between( {o_rkv, <<"Bucket">>, null, null}, {o_rkv, <<"Bucket">>, null, null}, Tree, - fun leveled_pmanifest:entry_startkey/1 + EndRangeFun ), ?assertMatch(1, length(R)). diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 65c3c7f1..8eb76174 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -11,6 +11,7 @@ space_clear_ondelete/1, is_empty_test/1, many_put_fetch_switchcompression/1, + many_put_fetch_switchledgerversion/1, bigjournal_littlejournal/1, bigsst_littlesst/1, safereaderror_startup/1, @@ -30,6 +31,7 @@ all() -> space_clear_ondelete, is_empty_test, many_put_fetch_switchcompression, + many_put_fetch_switchledgerversion, bigjournal_littlejournal, bigsst_littlesst, safereaderror_startup, @@ -1395,14 +1397,49 @@ remove_journal_test(_Config) -> many_put_fetch_switchcompression(_Config) -> {T0, ok} = - timer:tc(fun many_put_fetch_switchcompression_tester/1, [native]), + timer:tc( + fun many_put_fetch_switch_tester/1, + [set_compression_start_opts(native)] + ), {T1, ok} = - timer:tc(fun many_put_fetch_switchcompression_tester/1, [lz4]), + timer:tc( + fun many_put_fetch_switch_tester/1, + [set_compression_start_opts(lz4)] + ), {T2, ok} = - timer:tc(fun many_put_fetch_switchcompression_tester/1, [zstd]), + timer:tc( + fun many_put_fetch_switch_tester/1, + [set_compression_start_opts(zstd)] + ), io:format("Test timings native=~w lz4=~w, zstd=~w", [T0, T1, T2]). -many_put_fetch_switchcompression_tester(CompressionMethod) -> +many_put_fetch_switchledgerversion(_Config) -> + {T0, ok} = + timer:tc( + fun many_put_fetch_switch_tester/1, + [set_ledgermd_version_start_opts(2, 3, 2)] + ), + {T1, ok} = + timer:tc( + fun many_put_fetch_switch_tester/1, + [set_ledgermd_version_start_opts(3, 2, 3)] + ), + {T2, ok} = + timer:tc( + fun many_put_fetch_switch_tester/1, + [set_ledgermd_version_start_opts(3, 3, 3)] + ), + {T3, ok} = + timer:tc( + fun many_put_fetch_switch_tester/1, + [set_ledgermd_version_start_opts(2, 2, 2)] + ), + io:format( + "Test timings switching ~w ~w all l3 ~w all l2 ~w~n", + [T0, T1, T2, T3] + ). + +set_compression_start_opts(CompressionMethod) -> RootPath = testutil:reset_filestructure(), StartOpts1 = [ {root_path, RootPath}, @@ -1429,7 +1466,40 @@ many_put_fetch_switchcompression_tester(CompressionMethod) -> {compression_method, none}, {ledger_compression, as_store} ], + {StartOpts1, StartOpts2, StartOpts3}. + +set_ledgermd_version_start_opts(SL, ML, EL) -> + RootPath = testutil:reset_filestructure(), + StartOpts1 = [ + {root_path, RootPath}, + {max_pencillercachesize, 16000}, + {max_journalobjectcount, 30000}, + {sync_strategy, testutil:sync_strategy()}, + {compression_method, zstd}, + {ledger_compression, as_store}, + {ledger_value_version, SL} + ], + StartOpts2 = [ + {root_path, RootPath}, + {max_pencillercachesize, 24000}, + {max_journalobjectcount, 30000}, + {sync_strategy, testutil:sync_strategy()}, + {compression_method, zstd}, + {ledger_compression, as_store}, + {ledger_value_version, ML} + ], + StartOpts3 = [ + {root_path, RootPath}, + {max_pencillercachesize, 16000}, + {max_journalobjectcount, 30000}, + {sync_strategy, testutil:sync_strategy()}, + {compression_method, zstd}, + {ledger_compression, as_store}, + {ledger_value_version, EL} + ], + {StartOpts1, StartOpts2, StartOpts3}. +many_put_fetch_switch_tester({StartOpts1, StartOpts2, StartOpts3}) -> {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec), @@ -1541,7 +1611,6 @@ many_put_fetch_switchcompression_tester(CompressionMethod) -> ok = leveled_bookie:book_close(Bookie4), - %% Change compression method -> lz4 {ok, Bookie5} = leveled_bookie:book_start(StartOpts2), lists:foreach( fun(CL) -> ok = testutil:check_forlist(Bookie5, CL) end, CL1s @@ -1552,9 +1621,12 @@ many_put_fetch_switchcompression_tester(CompressionMethod) -> lists:foreach( fun(CL) -> ok = testutil:check_forlist(Bookie5, CL) end, CL5s ), + + {Size5, Count5} = testutil:check_bucket_stats(Bookie5, <<"Bucket">>), + io:format("Stats ~w ~w from ~s~n", [Size5, Count5, <<"Bucket">>]), + ok = leveled_bookie:book_close(Bookie5), - %% Change compression method -> native {ok, Bookie6} = leveled_bookie:book_start(StartOpts1), lists:foreach( fun(CL) -> ok = testutil:check_forlist(Bookie6, CL) end, CL1s @@ -1566,6 +1638,13 @@ many_put_fetch_switchcompression_tester(CompressionMethod) -> fun(CL) -> ok = testutil:check_forlist(Bookie6, CL) end, CL5s ), + {Size6, Count6} = testutil:check_bucket_stats(Bookie6, <<"Bucket">>), + + true = Size5 == Size6, + true = Count5 == Count6, + true = Size5 > 0, + true = Count5 > 0, + ok = leveled_bookie:book_destroy(Bookie6). safereaderror_startup(_Config) -> diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 76238a07..51e2588d 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -40,6 +40,10 @@ end_per_suite(Config) -> testutil:end_per_suite(Config). expiring_indexes(_Config) -> + expiring_indexes_tester(2), + expiring_indexes_tester(3). + +expiring_indexes_tester(LVV) -> % Add objects to the store with index entries, where the objects (and hence % the indexes have an expiry time. Confirm that the indexes and the % objects are no longer present after the expiry time (and are present @@ -55,6 +59,7 @@ expiring_indexes(_Config) -> {root_path, RootPath}, {max_pencillercachesize, 16000}, {max_journalobjectcount, 30000}, + {ledger_value_version, LVV}, {sync_strategy, testutil:sync_strategy()} ], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), @@ -632,10 +637,13 @@ small_load_with2i(_Config) -> query_count(_Config) -> RootPath = testutil:reset_filestructure(), - {ok, Book1} = - leveled_bookie:book_start( - RootPath, 2000, 50000000, testutil:sync_strategy() - ), + StartOpts2 = [ + {root_path, RootPath}, + {max_journalsize, 50000000}, + {ledger_value_version, 2}, + {sync_strategy, testutil:sync_strategy()} + ], + {ok, Book1} = leveled_bookie:book_start(StartOpts2), BucketBin = list_to_binary("Bucket"), {TestObject, TestSpec} = testutil:generate_testobject( diff --git a/test/end_to_end/perf_SUITE.erl b/test/end_to_end/perf_SUITE.erl index 30beb87f..83615912 100644 --- a/test/end_to_end/perf_SUITE.erl +++ b/test/end_to_end/perf_SUITE.erl @@ -66,11 +66,11 @@ riak_fullperf(ObjSize, PM, LC) -> output_result(R2B), R2C = riak_load_tester(Bucket, 2000000, ObjSize, [], PM, LC), output_result(R2C), - R5A = riak_load_tester(Bucket, 5000000, ObjSize, [], PM, LC), + R5A = riak_load_tester(Bucket, 3000000, ObjSize, [], PM, LC), output_result(R5A), - R5B = riak_load_tester(Bucket, 5000000, ObjSize, [], PM, LC), + R5B = riak_load_tester(Bucket, 3000000, ObjSize, [], PM, LC), output_result(R5B), - R10 = riak_load_tester(Bucket, 8000000, ObjSize, [], PM, LC), + R10 = riak_load_tester(Bucket, 5000000, ObjSize, [], PM, LC), output_result(R10). riak_profileperf(_Config) -> @@ -407,7 +407,7 @@ profile_app(Pids, ProfiledFun, P) -> MinTime = case P of P when P == query; P == mini_query -> - 100000; + 120000; P when P == head; P == load -> 200000; _ -> diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 44141bc1..76b1d584 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -6,7 +6,8 @@ many_put_compare/1, index_compare/1, basic_headonly/1, - tuplebuckets_headonly/1 + tuplebuckets_headonly/1, + headonly_trim_and_key_rotation/1 ]). all() -> @@ -15,7 +16,8 @@ all() -> many_put_compare, index_compare, basic_headonly, - tuplebuckets_headonly + tuplebuckets_headonly, + headonly_trim_and_key_rotation ]. -define(V1_VERS, 1). @@ -36,14 +38,15 @@ multiput_subkeys(_Config) -> multiput_subkeys_byvalue(V) -> RootPath = testutil:reset_filestructure("subkeyTest"), - StartOpts = [ + StartOpts2 = [ {root_path, RootPath}, {max_journalsize, 10000000}, {max_pencillercachesize, 12000}, {head_only, no_lookup}, + {ledger_value_version, 2}, {sync_strategy, testutil:sync_strategy()} ], - {ok, Bookie} = leveled_bookie:book_start(StartOpts), + {ok, Bookie} = leveled_bookie:book_start(StartOpts2), SubKeyCount = 200000, B = {<<"MultiBucketType">>, <<"MultiBucket">>}, @@ -61,12 +64,23 @@ multiput_subkeys_byvalue(V) -> load_objectspecs(SpecL1, 32, Bookie), SpecL2 = ObjSpecLGen(<<2:32/integer>>), load_objectspecs(SpecL2, 32, Bookie), + + ok = leveled_bookie:book_close(Bookie), + StartOpts3 = + lists:keyreplace( + ledger_value_version, + 1, + StartOpts2, + {ledger_value_version, 3} + ), + {ok, Bookie3} = leveled_bookie:book_start(StartOpts3), + SpecL3 = ObjSpecLGen(<<3:32/integer>>), - load_objectspecs(SpecL3, 32, Bookie), + load_objectspecs(SpecL3, 32, Bookie3), SpecL4 = ObjSpecLGen(<<4:32/integer>>), - load_objectspecs(SpecL4, 32, Bookie), + load_objectspecs(SpecL4, 32, Bookie3), SpecL5 = ObjSpecLGen(<<5:32/integer>>), - load_objectspecs(SpecL5, 32, Bookie), + load_objectspecs(SpecL5, 32, Bookie3), FoldFun = fun(Bucket, {Key, SubKey}, _Value, Acc) -> @@ -76,11 +90,17 @@ multiput_subkeys_byvalue(V) -> end end, QueryFun = - fun(KeyRange) -> + fun(KeyRange, CurrentBookie) -> Range = {range, B, KeyRange}, {async, R} = leveled_bookie:book_headfold( - Bookie, ?HEAD_TAG, Range, {FoldFun, []}, false, true, false + CurrentBookie, + ?HEAD_TAG, + Range, + {FoldFun, []}, + false, + true, + false ), L = length(R()), io:format("query result for range ~p is ~w~n", [Range, L]), @@ -94,10 +114,18 @@ multiput_subkeys_byvalue(V) -> {<<1:32/integer>>, <<10:32/integer>>}, {<<2:32/integer>>, <<19:32/integer>>} }, - true = SubKeyCount == QueryFun(KR1), - true = (SubKeyCount * 2) == QueryFun(KR2), - true = (SubKeyCount + 10) == QueryFun(KR3), - leveled_bookie:book_destroy(Bookie). + true = SubKeyCount == QueryFun(KR1, Bookie3), + true = (SubKeyCount * 2) == QueryFun(KR2, Bookie3), + true = (SubKeyCount + 10) == QueryFun(KR3, Bookie3), + + leveled_bookie:book_close(Bookie3), + {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), + + true = SubKeyCount == QueryFun(KR1, Bookie2), + true = (SubKeyCount * 2) == QueryFun(KR2, Bookie2), + true = (SubKeyCount + 10) == QueryFun(KR3, Bookie2), + + leveled_bookie:book_destroy(Bookie2). many_put_compare(_Config) -> TreeSize = small, @@ -137,7 +165,7 @@ many_put_compare(_Config) -> {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), testutil:check_forobject(Bookie2, TestObject), - % Generate 200K objects to be sued within the test, and load them into + % Generate 200K objects to be used within the test, and load them into % the first store (outputting the generated objects as a list of lists) % to be used elsewhere @@ -866,6 +894,74 @@ tuplebuckets_headonly(_Config) -> leveled_bookie:book_destroy(Bookie1). +headonly_trim_and_key_rotation(_Config) -> + %% Rotate a small set of keys, and ensure that trim still has an impact + %% See - https://github.com/martinsumner/leveled/issues/497 + RootPathHO = testutil:reset_filestructure("trimHO"), + StartOpts1 = [ + {root_path, RootPathHO}, + {max_pencillercachesize, 8000}, + {cache_size, 2000}, + {head_only, no_lookup}, + {max_journalobjectcount, 5000} + ], + + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + + ObjectSpecFun = + fun(Op, V) -> + fun(N) -> + Bucket = <<"B", N:8/integer>>, + Key = <<"K", N:32/integer>>, + <> = + crypto:hash(md5, term_to_binary({Bucket, Key})), + {Op, v1, <>, Bucket, Key, undefined, + {value, V}} + end + end, + + LoadFun = + fun(Book, V, Cnt) -> + ST = os:system_time(millisecond), + ObjectSpecL = lists:map(ObjectSpecFun(add, V), lists:seq(1, Cnt)), + ok = load_objectspecs(ObjectSpecL, 8, Book), + io:format( + "ObjectSpec load of ~w took ~w ms~n", + [Cnt, os:system_time(millisecond) - ST] + ) + end, + + lists:foreach(fun(I) -> LoadFun(Bookie1, I, 1000) end, lists:seq(1, 2000)), + + JFP = RootPathHO ++ "/journal/journal_files", + {ok, FNs} = file:list_dir(JFP), + + io:format("Journal file count of ~w discovered~n", [length(FNs)]), + ok = leveled_bookie:book_trimjournal(Bookie1), + + WaitForTrimFun = + fun + (N, false) -> + {ok, PollFNs} = file:list_dir(JFP), + io:format( + "Journal files count discovered after trim triggered ~w~n", + [length(PollFNs)] + ), + case length(PollFNs) < length(FNs) of + true -> + true; + false -> + timer:sleep(N * 1000), + false + end; + (_N, true) -> + true + end, + + true = lists:foldl(WaitForTrimFun, false, [1, 2, 3, 5, 8, 13, 21]), + + ok = leveled_bookie:book_destroy(Bookie1). + basic_headonly(_Config) -> ObjectCount = 200000, RemoveCount = 100, @@ -963,7 +1059,7 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> {ok, FinalFNs} = file:list_dir(JFP), ok = leveled_bookie:book_trimjournal(Bookie1), - % CCheck a second trim is still OK + % Check a second trim is still OK [{add, SegmentID0, Bucket0, Key0, Hash0} | _Rest] = ObjectSpecL, case HeadOnly of diff --git a/test/property/codec_eqc.erl b/test/property/codec_eqc.erl new file mode 100644 index 00000000..5eb8d6c4 --- /dev/null +++ b/test/property/codec_eqc.erl @@ -0,0 +1,243 @@ + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% leveled_codec Binary Encoding Specification +%% +%% All integers are big-endian unsigned unless otherwise noted. +%% All fields are contiguous with no padding or alignment. +%% "EXT" denotes Erlang External Term Format (OTP term_to_binary). +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +%% ============================================================== +%% 1. Ledger Value +%% +%% Historically stored as Erlang tuples (versions 1 and 2). +%% Version 3 uses the pure-binary format below. +%% ============================================================== + +%% ledger-value = ledger-value-v1 ; legacy: 4-tuple {sqn,status,hash,md} +%% / ledger-value-v2 ; legacy: 5-tuple {sqn,status,hash,md,lmd} +%% / ledger-value-v3 ; current binary format +%% +%% ledger-value-v3 = LV3-VERSION +%% LV3-STATUS +%% LV3-SEG-HASH +%% LV3-LMD +%% LV3-SQN +%% LV3-UMD +%% +%% LV3-VERSION = %x03 ; literal version tag +%% +%% LV3-STATUS = status-active-inf +%% / status-tomb +%% / status-active-ttl +%% +%% status-active-inf = %x00 ; high-nibble=0 (active), low-nibble=0 (no TTL length) +%% status-tomb = %x10 ; high-nibble=1 (tomb), low-nibble=0 +%% +%% TTL: high-nibble=2, low-nibble=L, followed by L bytes of timestamp +%% status-active-ttl = ttl-header *OCTET ; 1*15OCTET length is low-nibble of ttl-header +%% ttl-header = %x21-2F ; byte = (0x2 << 4) | L, L in 1..15 +%% ; L = byte_size(binary:encode_unsigned(TTL)) +%% ; TTL expressed as seconds since Unix epoch +%% +%% LV3-SEG-HASH = seg-hash-present +%% / seg-hash-absent +%% +%% seg-hash-present = %x00 seg-hash-lo extra-hash +%% seg-hash-lo = 2OCTET ; 16-bit segment hash +%% extra-hash = 4OCTET ; 32-bit extra hash +%% +%% seg-hash-absent = %x01 ; no-lookup marker (index entries) +%% +%% LV3-LMD = lmd-absent +%% / lmd-present +%% +%% lmd-absent = %x00 ; undefined / not recorded +%% lmd-present = lmd-length lmd-value +%% lmd-length = OCTET ; L in 1..4 (epoch seconds fit in 4 bytes +%% ; until year 2106; encoded as minimum bytes) +%% lmd-value = 1*255OCTET ; big-endian unsigned integer, L bytes +%% +%% LV3-SQN = sqn-length sqn-value +%% sqn-length = OCTET ; L = byte_size(binary:encode_unsigned(SQN)) +%% sqn-value = 1*255OCTET ; big-endian unsigned integer, L bytes +%% +%% %% LV3-UMD = umd-absent +%% / umd-present +%% +%% umd-absent = %x00 +%% umd-present = %x01 umd-length umd-bytes ; NOTE seems not needed with %x01, see lmd +%% umd-length = 3OCTET ; 24-bit big-endian byte count N, N < 16777216 +%% umd-bytes = 1*OCTET ; N bytes of Erlang EXT (term_to_binary/1) +%% ; WARNING: no size guard in create_v3_value +%% ; if byte_size(EXT) >= 2^24 the encoder crashes + +-module(codec_eqc). + +-ifdef(EQC). + +-compile([export_all, nowarn_export_all]). + +-include_lib("eqc/include/eqc.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include("../include/leveled.hrl"). + +-define(QC_OUT(P), + eqc:on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). + +eqc_prop_versions_test_() -> + {timeout, + ?EQC_TIME_BUDGET + 10, + ?_assertEqual( + true, + eqc:quickcheck( + eqc:testing_time(?EQC_TIME_BUDGET div 2, ?QC_OUT(prop_value_versions()))))}. + +eqc_prop_v3_binary_test_() -> + {timeout, + ?EQC_TIME_BUDGET + 10, + ?_assertEqual( + true, + eqc:quickcheck( + eqc:testing_time(?EQC_TIME_BUDGET div 2, ?QC_OUT(prop_v3_binary()))))}. + + +%% generators + +pos() -> + choose(1, 16#ffff). + +ledger_status() -> + oneof([tomb, {active, nat()}, {active, infinity}]). + +ledger_seg_hash() -> + oneof([no_lookup, {choose(0, 16#ffff), choose(0, 16#ffff)}]). + +ledger_metadata() -> + %% for any() take just int() or bool() for the moment + oneof([{int(), int()}, int(), bool(), atom, binary()]). + +ledger_last_moddate() -> + oneof([undefined, ?LET(N, choose(-16#ffff, 16#ffff), N + 1786520336)]). + +%% Grammar-based generators for ledger-value-v3 binary fields. +%% Each generator corresponds directly to an ABNF rule in the spec above. + +%% LV3-STATUS = status-active-inf / status-tomb / status-active-ttl +gen_lv3_status() -> + oneof([ + return(<<0:8>>), %% status-active-inf = %x00 + return(<<1:4, 0:4>>), %% status-tomb = %x10 + ?LET(L, choose(1, 15), %% status-active-ttl = ttl-header *OCTET + ?LET(Bytes, binary(L), + <<2:4, L:4, Bytes/binary>>)) + ]). + +%% LV3-SEG-HASH = seg-hash-present / seg-hash-absent +gen_lv3_seg_hash() -> + oneof([ + return(<<1:8>>), %% seg-hash-absent = %x01 + ?LET({SH, EH}, %% seg-hash-present = %x00 seg-hash-lo extra-hash + {choose(0, 16#ffff), choose(0, 16#ffffffff)}, + <<0:8, SH:16, EH:32>>) + ]). + +%% LV3-LMD = lmd-absent / lmd-present +gen_lv3_lmd() -> + oneof([ + return(<<0:8>>), %% lmd-absent = %x00 + ?LET(L, choose(1, 255), %% lmd-present = lmd-length lmd-value + ?LET(Bytes, binary(L), + <>)) + ]). + +%% LV3-SQN = sqn-length sqn-value +gen_lv3_sqn() -> + ?LET(L, choose(1, 255), + ?LET(Bytes, binary(L), + <>)). + +%% LV3-UMD = umd-absent / umd-present +gen_lv3_umd() -> + oneof([ + return(<<0:8>>), %% umd-absent = %x00 + ?LET(Term, ledger_metadata(), %% umd-present = %x1(umd-length):4 umd-bytes + begin + UMDBin = term_to_binary(Term), + UmdSize = byte_size(UMDBin), + Bytes = byte_size(binary:encode_unsigned(UmdSize)), + <<1:4, Bytes:4, UmdSize:(Bytes*8), UMDBin/binary>> + end) + ]). + +%% ledger-value-v3 = LV3-VERSION LV3-STATUS LV3-SEG-HASH LV3-LMD LV3-SQN LV3-UMD +v3_binary() -> + {gen_lv3_status(), gen_lv3_seg_hash(), gen_lv3_lmd(), gen_lv3_sqn(), gen_lv3_umd()}. + + + +%% From type definition in leveled_codec.erl: +%% -type ledger_value_v2() :: {sqn(), ledger_status(), segment_hash(), metadata(), last_moddate()}. +prop_value_versions() -> + ?FORALL( + {Sqn, Status, SegHash, MD, Lmd}, + {pos(), ledger_status(), ledger_seg_hash(), ledger_metadata(), ledger_last_moddate()}, + begin + VV1 = {Sqn, Status, SegHash, MD}, + VV2 = {Sqn, Status, SegHash, MD, Lmd}, + VV3 = leveled_codec:create_v3_value(Sqn, Status, SegHash, MD, Lmd), + conjunction([ + {sqn1, equals(leveled_codec:ledgermd_sqn(VV1), Sqn)}, + {sqn2, equals(leveled_codec:ledgermd_sqn(VV2), Sqn)}, + {sqn3, equals(leveled_codec:ledgermd_sqn(VV3), Sqn)}, + {seg_hash1, equals(leveled_codec:ledgermd_seg(VV1), SegHash)}, + {seg_hash2, equals(leveled_codec:ledgermd_seg(VV2), SegHash)}, + {seg_hash3, equals(leveled_codec:ledgermd_seg(VV3), SegHash)}, + {seg_hashlmd2, equals(leveled_codec:ledgermd_seglmd(VV2), {SegHash, Lmd})}, + {seg_hashlmd3, equals(leveled_codec:ledgermd_seglmd(VV3), {SegHash, Lmd})}, + {status_and_sqn1, equals(leveled_codec:ledgermd_statussqn(VV1), {Status, Sqn})}, + {status_and_sqn2, equals(leveled_codec:ledgermd_statussqn(VV2), {Status, Sqn})}, + {status_and_sqn3, equals(leveled_codec:ledgermd_statussqn(VV3), {Status, Sqn})}, + {status_lmd2, equals(leveled_codec:ledgermd_statuslmd(VV2), {Status, Lmd})}, + {status_lmd3, equals(leveled_codec:ledgermd_statuslmd(VV3), {Status, Lmd})}, + {status_sqn_umd1, equals(leveled_codec:ledgermd_statussqnumd(VV1), {Status, Sqn, MD})}, + {status_sqn_umd2, equals(leveled_codec:ledgermd_statussqnumd(VV2), {Status, Sqn, MD})}, + {status_sqn_umd3, equals(leveled_codec:ledgermd_statussqnumd(VV3), {Status, Sqn, MD})}, + {sqn_umd1, equals(leveled_codec:ledgermd_sqnumd(VV1), {Sqn, MD})}, + {sqn_umd2, equals(leveled_codec:ledgermd_sqnumd(VV2), {Sqn, MD})}, + {sqn_umd3, equals(leveled_codec:ledgermd_sqnumd(VV3), {Sqn, MD})}, + {umd1, equals(leveled_codec:ledgermd_umd(VV1), MD)}, + {umd2, equals(leveled_codec:ledgermd_umd(VV2), MD)}, + {umd3, equals(leveled_codec:ledgermd_umd(VV3), MD)}, + {status1, equals(leveled_codec:ledgermd_status(VV1), Status)}, + {status2, equals(leveled_codec:ledgermd_status(VV2), Status)}, + {status3, equals(leveled_codec:ledgermd_status(VV3), Status)} + ]) + end). + +%% We encode v3 binaries from the grammar directly to spot future changes in decoder. +%% Note that different binaries can be decoded to the sae value, for example a +%% sequence number can have additional leading zeros ( 1 0 and 2 0 0 are one and two byte representation of zero) +prop_v3_binary() -> + ?FORALL( + {StatusBin, SegHashBin, LmdBin, SqnBin, UmdBin}, + v3_binary(), + begin + V3Binary = <<3:8, StatusBin/binary, SegHashBin/binary, LmdBin/binary, SqnBin/binary, UmdBin/binary>>, + %% Test no crash on decoding any v3 binary, even if it is invalid. + Sqn = leveled_codec:ledgermd_sqn(V3Binary), + Status = leveled_codec:ledgermd_status(V3Binary), + SegHash = leveled_codec:ledgermd_seg(V3Binary), + MD = leveled_codec:ledgermd_umd(V3Binary), + {SegHash, Lmd} = leveled_codec:ledgermd_seglmd(V3Binary), + {Status, Lmd} = leveled_codec:ledgermd_statuslmd(V3Binary), + {Status, Sqn} = leveled_codec:ledgermd_statussqn(V3Binary), + {Status, Sqn, MD} = leveled_codec:ledgermd_statussqnumd(V3Binary), + {Sqn, MD} = leveled_codec:ledgermd_sqnumd(V3Binary), + true + end). + + +-endif.